Merge branch 'main' into cmoyes/elfmismatch

Author: cmoyes-jump

.github/workflows/check_seccomp.yml

@@ -0,0 +1,33 @@
+name: Check Generated Seccomp Files
+on:
+  pull_request:
+
+permissions: {}
+
+jobs:
+  check-seccomp:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+
+    steps:
+      - name: Checkout current commit
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python environment
+        run: |
+          python -m venv venv && venv/bin/pip install edn_format
+
+      - name: Check generated seccomp files
+        run: |
+          set -e
+          source venv/bin/activate
+          make seccomp-policies
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "Generated seccomp files are out of date. Please run 'make seccomp-policies' and commit the changes."
+            git --no-pager diff
+            exit 1
+          else
+            echo "All generated seccomp files are up to date."
+          fi

NOTICE

@@ -545,3 +545,24 @@ https://github.com/solana-program/token/tree/program%40v8.0.0/p-token
                         http://www.apache.org/licenses/
 
 =======================================================================
+
+The flatbuffers generated code in src/flamenco/runtime/tests/flatbuffers/
+is generated by flatcc.
+
+https://github.com/dvidelabs/flatcc/
+
+Copyright 2015 Mikkel F. Jørgensen, dvide.com
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+=======================================================================

config/extra/with-arm.mk

@@ -34,6 +34,7 @@ include config/extra/with-bzip2.mk
 include config/extra/with-lz4.mk
 include config/extra/with-openssl.mk
 include config/extra/with-rocksdb.mk
+include config/extra/with-flatcc.mk
 
 endif
 

config/extra/with-flatcc.mk

@@ -0,0 +1,7 @@
+ifneq (,$(wildcard $(OPT)/lib/libflatccrt.a))
+FLATCC_LIBS:=$(OPT)/lib/libflatccrt.a
+FD_HAS_FLATCC:=1
+CPPFLAGS+=-DFD_HAS_FLATCC=1
+else
+$(info "flatcc not installed, skipping")
+endif

config/extra/with-liburing.mk

@@ -0,0 +1,7 @@
+ifneq (,$(wildcard $(OPT)/lib/liburing.a))
+FD_HAS_LIBURING:=1
+CFLAGS+=-DFD_HAS_LIBURING=1
+LDFLAGS+=$(OPT)/lib/liburing.a
+else
+$(warning "liburing not installed, skipping")
+endif

config/extra/with-x86-64.mk

@@ -16,9 +16,11 @@ include config/extra/with-ucontext.mk
 include config/extra/with-secp256k1.mk
 include config/extra/with-s2nbignum.mk
 include config/extra/with-zstd.mk
+include config/extra/with-liburing.mk
 include config/extra/with-bzip2.mk
 include config/extra/with-lz4.mk
 include config/extra/with-openssl.mk
 include config/extra/with-rocksdb.mk
+include config/extra/with-flatcc.mk
 
 FD_ARCH_SUPPORTS_SANDBOX:=1

deps.sh

@@ -35,6 +35,7 @@ PREFIX="$(pwd)/opt"
 
 DEVMODE=0
 MSAN=0
+LIBURING=0
 _CC="${CC:=gcc}"
 _CXX="${CXX:=g++}"
 EXTRA_CFLAGS="-g3 -fno-omit-frame-pointer"
@@ -133,15 +134,17 @@ fetch () {
   if [[ $MSAN == 1 ]]; then
     checkout_llvm
   fi
-  checkout_repo zstd      https://github.com/facebook/zstd          "v1.5.7"
-  checkout_repo lz4       https://github.com/lz4/lz4                "v1.10.0"
-  checkout_repo s2n       https://github.com/awslabs/s2n-bignum     "" "4d2e22a"
-  checkout_repo openssl   https://github.com/openssl/openssl        "openssl-3.6.0"
-  checkout_repo secp256k1 https://github.com/bitcoin-core/secp256k1 "v0.7.0"
+  checkout_repo zstd      https://github.com/facebook/zstd            "v1.5.7"
+  checkout_repo lz4       https://github.com/lz4/lz4                  "v1.10.0"
+  checkout_repo liburing  https://github.com/axboe/liburing           "liburing-2.12"
+  checkout_repo s2n       https://github.com/awslabs/s2n-bignum       "" "4d2e22a"
+  checkout_repo openssl   https://github.com/openssl/openssl          "openssl-3.6.0"
+  checkout_repo secp256k1 https://github.com/bitcoin-core/secp256k1   "v0.7.0"
+  checkout_repo flatcc    https://github.com/dvidelabs/flatcc.git     "" "3ae5eda"
   if [[ $DEVMODE == 1 ]]; then
-    checkout_repo bzip2     https://gitlab.com/bzip2/bzip2            "bzip2-1.0.8"
-    checkout_repo rocksdb   https://github.com/facebook/rocksdb       "v10.5.1"
-    checkout_repo snappy    https://github.com/google/snappy          "1.2.2"
+    checkout_repo bzip2   https://gitlab.com/bzip2/bzip2              "bzip2-1.0.8"
+    checkout_repo rocksdb https://github.com/facebook/rocksdb         "v10.5.1"
+    checkout_repo snappy  https://github.com/google/snappy            "1.2.2"
   fi
 }
 
@@ -438,6 +441,16 @@ install_lz4 () {
   echo "[+] Successfully installed lz4"
 }
 
+install_liburing () {
+  cd "$PREFIX/git/liburing"
+
+  echo "[+] Installing liburing to $PREFIX"
+  ./configure --prefix="$PREFIX" --cc="$CC -fPIC $EXTRA_CFLAGS"
+  "${MAKE[@]}"
+  "${MAKE[@]}" install
+  echo "[+] Successfully installed liburing"
+}
+
 install_s2n () {
   cd "$PREFIX/git/s2n"
 
@@ -610,6 +623,21 @@ install_snappy () {
   echo "[+] Successfully installed snappy"
 }
 
+install_flatcc () {
+  echo "[+] Installing flatcc"
+  cd "$PREFIX/git/flatcc"
+  cmake -B build \
+    -DCMAKE_INSTALL_PREFIX=$PREFIX \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DFLATCC_INSTALL=ON \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+    -DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS" \
+    -DCMAKE_EXE_LINKER_FLAGS="$EXTRA_LDFLAGS"
+  cmake --build build -j
+  cmake --install build
+  echo "[+] Successfully installed flatcc"
+}
+
 install () {
   CC="$(command -v $_CC)"
   cc="$CC"
@@ -629,6 +657,11 @@ install () {
   fi
   ( install_zstd      )
   ( install_lz4       )
+  if [[ $LIBURING == 1 ]]; then
+    if [[ "$OS" == "Linux" ]]; then
+      ( install_liburing )
+    fi
+  fi
   if [[ "$(uname -m)" == x86_64 ]]; then
     ( install_s2n )
   fi
@@ -638,6 +671,7 @@ install () {
     ( install_bzip2     )
     ( install_snappy    )
     ( install_rocksdb   )
+    ( install_flatcc    )
   fi
 
   # Merge lib64 with lib
@@ -673,6 +707,10 @@ while [[ $# -gt 0 ]]; do
       shift
       DEVMODE=1
       ;;
+    "+uring")
+      shift
+      LIBURING=1
+      ;;
     nuke)
       shift
       nuke

src/app/firedancer-dev/commands/snapshot_load.c

@@ -215,12 +215,6 @@ snapshot_load_topo( config_t * config,
 
     fd_topob_wksp( topo, "vinyl_exec" );
     fd_topo_tile_t * vinyl_tile = fd_topob_tile( topo, "vinyl", "vinyl_exec", "metric_in", ULONG_MAX, 0, 0 );
-    vinyl_tile->vinyl.vinyl_meta_map_obj_id  = fd_pod_query_ulong( topo->props, "vinyl.meta_map",  ULONG_MAX );
-    vinyl_tile->vinyl.vinyl_meta_pool_obj_id = fd_pod_query_ulong( topo->props, "vinyl.meta_pool", ULONG_MAX );
-    vinyl_tile->vinyl.vinyl_line_max         = config->firedancer.vinyl.max_cache_entries;
-    vinyl_tile->vinyl.vinyl_cnc_obj_id       = vinyl_cnc->id;
-    vinyl_tile->vinyl.vinyl_data_obj_id      = vinyl_data->id;
-    fd_cstr_ncpy( vinyl_tile->vinyl.vinyl_bstream_path, config->paths.accounts, sizeof(vinyl_tile->vinyl.vinyl_bstream_path) );
 
     fd_topob_tile_uses( topo, vinyl_tile, vinyl_cnc,  FD_SHMEM_JOIN_MODE_READ_WRITE );
     fd_topob_tile_uses( topo, vinyl_tile, vinyl_data, FD_SHMEM_JOIN_MODE_READ_WRITE );
@@ -583,6 +577,11 @@ fixup_config( config_t *     config,
 
     config->firedancer.funk.heap_size_gib       = 0;
     config->firedancer.funk.max_account_records = 0;
+
+    char const * io_mode = args->snapshot_load.vinyl_io;
+    if(      0==strcmp( io_mode, "ur" ) ) config->firedancer.vinyl.io_uring.enabled = 1;
+    else if( 0==strcmp( io_mode, "bd" ) ) {}
+    else FD_LOG_ERR(( "unsupported --vinyl-io '%s' (valid options are 'bd' and 'ur')", io_mode ));
   }
 
   if( args->snapshot_load.offline ) {
@@ -851,15 +850,15 @@ snapshot_load_cmd_fn( args_t *   args,
     /* Wait for vinyl tile to boot */
     fd_cnc_t * cnc = fd_cnc_join( fd_topo_obj_laddr( topo, fd_pod_query_ulong( topo->props, "vinyl.cnc", ULONG_MAX )  ) );
     FD_TEST( cnc );
-    ulong vinyl_status = fd_cnc_wait( cnc, FD_CNC_SIGNAL_BOOT, LONG_MAX, NULL );
-    FD_TEST( vinyl_status==FD_CNC_SIGNAL_RUN );
+    ulong vinyl_status = fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_BOOT, LONG_MAX, NULL );
+    FD_TEST( vinyl_status==FD_VINYL_CNC_SIGNAL_RUN );
     FD_LOG_NOTICE(( "Vinyl server running" ));
     for(;;) {
       vinyl_status = fd_cnc_wait( cnc, vinyl_status, LONG_MAX, NULL );
       char cnc_signal_cstr[ FD_VINYL_CNC_SIGNAL_CSTR_BUF_MAX ];
       fd_vinyl_cnc_signal_cstr( vinyl_status, cnc_signal_cstr );
       FD_LOG_NOTICE(( "Vinyl CNC signal %s", cnc_signal_cstr ));
-      if( vinyl_status==FD_CNC_SIGNAL_BOOT ) break;
+      //if( vinyl_status==FD_VINYL_CNC_SIGNAL_BOOT ) break;
     }
     FD_LOG_NOTICE(( "Vinyl server shut down" ));
     fd_cnc_leave( cnc );

src/app/firedancer/config/default.toml

@@ -504,6 +504,9 @@ user = ""
     file_size_gib = 16
     max_cache_entries = 1_000_000
     cache_size_gib = 2
+    [vinyl.io_uring]
+        enabled = false
+        queue_depth = 2048
 
 [runtime]
     # TODO: This is not respected, the max vote accounts seems to be

src/app/firedancer/topology.c

@@ -24,6 +24,7 @@
 #include "../../discof/restore/utils/fd_ssmsg.h"
 #include "../../flamenco/progcache/fd_progcache_admin.h"
 #include "../../vinyl/meta/fd_vinyl_meta.h"
+#include "../../vinyl/io/fd_vinyl_io.h" /* FD_VINYL_IO_TYPE_* */
 
 #include <sys/random.h>
 #include <sys/types.h>
@@ -673,11 +674,7 @@ fd_topo_initialize( config_t * config ) {
 
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "dedup_resolv", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "replay_exec",  0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
-  /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "genesi_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "replay_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
-  if( snapshots_enabled ) {
-                       fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "snapin_manif", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
-  }
   /**/                 fd_topob_tile_out(   topo, "tower",   0UL,                       "tower_out",    0UL                                                );
 
   /**/                 fd_topob_tile_in (   topo, "send",    0UL,          "metric_in", "replay_stake", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
@@ -879,12 +876,14 @@ fd_topo_initialize( config_t * config ) {
       config->firedancer.funk.max_database_transactions,
       config->firedancer.funk.heap_size_gib );
   /**/                 fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); /* TODO: Should be readonly? */
+  /**/                 fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "tower", 0UL  ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
   FOR(exec_tile_cnt)   fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec",   i   ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   FOR(bank_tile_cnt)   fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "bank",   i   ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   FOR(resolv_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "resolv", i   ) ], funk_obj, FD_SHMEM_JOIN_MODE_READ_ONLY  );
 
   fd_topo_obj_t * banks_obj = setup_topo_banks( topo, "banks", config->firedancer.runtime.max_live_slots, config->firedancer.runtime.max_fork_width, config->development.bench.larger_max_cost_per_block );
   /**/                 fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); /* TODO: Should be readonly? */
+  /**/                 fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "tower",  0UL ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
   FOR(exec_tile_cnt)   fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec",   i   ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); /* TODO: Should be readonly? */
   FOR(bank_tile_cnt)   fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "bank",   i   ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   FOR(resolv_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "resolv", i   ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_ONLY  );
@@ -1404,6 +1403,17 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "vinyl" ) ) ) {
 
+    tile->vinyl.vinyl_meta_map_obj_id  = fd_pod_query_ulong( config->topo.props, "vinyl.meta_map",  ULONG_MAX );
+    tile->vinyl.vinyl_meta_pool_obj_id = fd_pod_query_ulong( config->topo.props, "vinyl.meta_pool", ULONG_MAX );
+    tile->vinyl.vinyl_line_max         = config->firedancer.vinyl.max_cache_entries;
+    tile->vinyl.vinyl_cnc_obj_id       = fd_pod_query_ulong( config->topo.props, "vinyl.cnc",       ULONG_MAX );
+    tile->vinyl.vinyl_data_obj_id      = fd_pod_query_ulong( config->topo.props, "vinyl.data",      ULONG_MAX );
+    fd_cstr_ncpy( tile->vinyl.vinyl_bstream_path, config->paths.accounts, sizeof(tile->vinyl.vinyl_bstream_path) );
+
+    tile->vinyl.io_type = config->firedancer.vinyl.io_uring.enabled ?
+        FD_VINYL_IO_TYPE_UR : FD_VINYL_IO_TYPE_BD;
+    tile->vinyl.uring_depth = config->firedancer.vinyl.io_uring.queue_depth;
+
   } else {
     FD_LOG_ERR(( "unknown tile name `%s`", tile->name ));
   }