Merge branch 'main' of https://github.com/firedancer-io/firedancer into ibhatt/cleanup

Author: ibhatt-jumptrading

book/api/metrics-generated.md

@@ -702,13 +702,13 @@
 | <span class="metrics-name">repair_&#8203;sent_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;sent_&#8203;request_&#8203;types="<span class="metrics-enum">needed_&#8203;highest_&#8203;window</span>"} | counter | What types of client messages are we sending (Need Highest Window) |
 | <span class="metrics-name">repair_&#8203;sent_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;sent_&#8203;request_&#8203;types="<span class="metrics-enum">needed_&#8203;orphan</span>"} | counter | What types of client messages are we sending (Need Orphans) |
 | <span class="metrics-name">repair_&#8203;sent_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;sent_&#8203;request_&#8203;types="<span class="metrics-enum">pong</span>"} | counter | What types of client messages are we sending (Pong) |
-| <span class="metrics-name">repair_&#8203;send_&#8203;pkt_&#8203;rate</span> | gauge | Rate of network packets we are sending, in packets per second coarsely |
 | <span class="metrics-name">repair_&#8203;repaired_&#8203;slots</span> | counter | Until which slots have we fully repaired |
 | <span class="metrics-name">repair_&#8203;current_&#8203;slot</span> | counter | Our view of the current cluster slot, max slot received |
 | <span class="metrics-name">repair_&#8203;request_&#8203;peers</span> | counter | How many peers have we requested |
 | <span class="metrics-name">repair_&#8203;sign_&#8203;tile_&#8203;unavail</span> | counter | How many times no sign tiles were available to send request |
 | <span class="metrics-name">repair_&#8203;eager_&#8203;repair_&#8203;aggresses</span> | counter | How many times we pass eager repair threshold |
 | <span class="metrics-name">repair_&#8203;rerequest_&#8203;queue</span> | counter | How many times we re-request a shred from the inflights queue |
+| <span class="metrics-name">repair_&#8203;malformed_&#8203;ping</span> | counter | How many times we received a malformed ping |
 | <span class="metrics-name">repair_&#8203;slot_&#8203;complete_&#8203;time</span> | histogram | Time in seconds it took to complete a slot |
 | <span class="metrics-name">repair_&#8203;response_&#8203;latency</span> | histogram | Time in nanoseconds it took to receive a repair request response |
 | <span class="metrics-name">repair_&#8203;sign_&#8203;duration_&#8203;seconds</span> | histogram | Duration of signing a message |
@@ -1120,3 +1120,13 @@
 | <span class="metrics-name">benchs_&#8203;transactions_&#8203;sent</span> | counter | Number of benchmark packets sent |
 
 </div>
+
+## Snapwh Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">snapwh_&#8203;state</span> | gauge | State of the tile. 0=IDLE, 1=PROCESSING, 4=SHUTDOWN |
+
+</div>

book/api/websocket.md

@@ -1417,7 +1417,8 @@ identity is no longer in these three data sources, it will be removed.
                 "name": "ExampleStake Firedancer 🔥💃",
                 "details": "A longer description of the validator, perhaps describing the team behind it or how the node is operated",
                 "website": "https://github.com/firedancer-io/firedancer",
-                "icon_url": "https://docs.firedancer.io/fire.svg"
+                "icon_url": "https://docs.firedancer.io/fire.svg",
+                "keybase_username": ""
             }
         }
     ],
@@ -1449,29 +1450,30 @@ identity is no longer in these three data sources, it will be removed.
 | delinquent      | `boolean`      | Whether the vote account is delinquent or not. A vote account is considered delinquent if it has not had a vote land on chain for any of the last 127 (inclusive) confirmed slots, according to this validator. If there have been less than 128 confirmed slots on the chain (it is a new chain), a validator is considered delinquent only if it has not voted yet at all |
 
 **`PeerUpdateInfo`**
-| Field    | Type     | Description |
-|----------|----------|-------------|
-| name     | `string\|null` | Self reported name of the validator, could be any string or null if there is no name set |
-| details  | `string\|null` | Self reported detailed description of the validator, could be any string or null if there is no details set |
-| website  | `string\|null` | Self reported website of the validator, could be any string and need not be a valid URI, or could be null if there is no website set |
-| icon_url | `string\|null` | Self reported URL of the validator icon, could be any string and need not be a valid URI, or could be null if there is no icon URI set |
+| Field            | Type     | Description |
+|------------------|----------|-------------|
+| name             | `string` | Self reported name of the validator, could be any string or empty string if there is no name set |
+| details          | `string` | Self reported detailed description of the validator, could be any string or empty string if there is no details set |
+| website          | `string` | Self reported website of the validator, could be any string and need not be a valid URI, or could be empty string if there is no website set |
+| icon_url         | `string` | Self reported URL of the validator icon, could be any string and need not be a valid URI, or could be empty string if there is no icon URI set |
+| keybase_username | `string` | Self reported keybase username of the validator, could be any string or empty string if there is no username set.  Keybase is a public, legacy storage for icon images.  Although this method for publiscising an icon is deprecated, it is included for completeness as many validators have not migrated to `iconUrl` |
 
 **`PeerUpdate`**
-| Field    | Type   | Description
-|----------|--------|------------
-| identity | `string` | Identity public key of the validator, encoded in base58 |
-| gossip   | `PeerUpdateGossip\|null` | Information reported for the validator identity over the gossip network. This is authenticated and the gossip node must have been in possession of the private key to publish gossip data as this identity. Gossip information is not validated or checked for correctness and could be set to any values by the peer |
+| Field    | Type                      | Description |
+|----------|---------------------------|-------------|
+| identity | `string`                  | Identity public key of the validator, encoded in base58 |
+| gossip   | `PeerUpdateGossip\|null`  | Information reported for the validator identity over the gossip network. This is authenticated and the gossip node must have been in possession of the private key to publish gossip data as this identity. Gossip information is not validated or checked for correctness and could be set to any values by the peer |
 | vote     | `PeerUpdateVoteAccount[]` | Information about the vote account(s) associated with this identity key, if there are any. It is extremely unusual for multiple vote accounts to report the same identity key. Vote account information like stake and commission is derived from the accounts on chain and cannot be corrupt, invalid, or incorrect |
-| info     | `PeerUpdateInfo\|null` | If the validator has published self reported identifying information to the chain. This is authenticated and the operator must have been in possession of the private key to publish info as this identity. Information is not validated or checked for correctness and could be set to any values by the peer |
+| info     | `PeerUpdateInfo\|null`    | If the validator has published self reported identifying information to the chain. This is authenticated and the operator must have been in possession of the private key to publish info as this identity. Information is not validated or checked for correctness and could be set to any values by the peer |
 
 **`PeerRemove`**
-| Field    | Type   | Description |
-|----------|--------|-------------|
+| Field    | Type     | Description |
+|----------|----------|-------------|
 | identity | `string` | Identity public key of the validator, encoded in base58 |
 
 **`PeersUpdate`**
-| Field  | Type   | Description |
-|--------|--------|-------------|
+| Field  | Type                 | Description |
+|--------|----------------------|-------------|
 | add    | `GossipPeerUpdate[]` | List of peer validators that were added since the last update, or all of the peers for the first update after connecting |
 | update | `GossipPeerUpdate[]` | List of peer validators that were changed since the last update |
 | remove | `GossipPeerRemove[]` | List of peer validators that were removed since the last update |

config/extra/with-offline-replay.mk

@@ -1,4 +1,5 @@
 include config/extra/with-handholding.mk
+include config/extra/with-debug.mk
 CPPFLAGS+=-DFD_SPAD_TRACK_USAGE=1
 CFLAGS+=-DFD_SPAD_TRACK_USAGE=1
 

contrib/offline-replay/offline_replay.toml

@@ -10,6 +10,8 @@
         cluster_version = "{cluster_version}"
     [tiles.gui]
         enabled = false
+    [tiles.rpc]
+        enabled = false
 [funk]
     heap_size_gib = {funk_pages}
     max_account_records = {index_max}

deps.sh

@@ -665,9 +665,9 @@ while [[ $# -gt 0 ]]; do
       PREFIX="$(pwd)/opt-msan"
       _CC=clang
       _CXX=clang++
-      EXTRA_CFLAGS+="-fsanitize=memory"
-      EXTRA_CXXFLAGS+="$EXTRA_CFLAGS -nostdinc++ -nostdlib++ -isystem $PREFIX/include/c++/v1"
-      EXTRA_LDFLAGS+="$PREFIX/lib/libc++.a $PREFIX/lib/libc++abi.a"
+      EXTRA_CFLAGS+=" -fsanitize=memory"
+      EXTRA_CXXFLAGS+=" $EXTRA_CFLAGS -nostdinc++ -nostdlib++ -isystem $PREFIX/include/c++/v1"
+      EXTRA_LDFLAGS+=" $PREFIX/lib/libc++.a $PREFIX/lib/libc++abi.a"
       ;;
     "+dev")
       shift

src/app/firedancer-dev/commands/backtest.c

@@ -90,7 +90,7 @@ backtest_topo( config_t * config ) {
   fd_topob_tile_uses( topo, replay_tile, progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
 
   if( snap_vinyl ) {
-    setup_topo_vinyl( topo, &config->firedancer );
+    setup_topo_vinyl_meta( topo, &config->firedancer );
   }
 
   /**********************************************************************/
@@ -294,9 +294,6 @@ backtest_topo( config_t * config ) {
   for( ulong i=0UL; i<exec_tile_cnt; i++ ) {
     fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   }
-  for( ulong i=0UL; i<bank_tile_cnt; i++ ) {
-    fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "bank", i ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
-  }
 
   FD_TEST( fd_pod_insertf_ulong( topo->props, txncache_obj->id, "txncache" ) );
   for( ulong i=0UL; i<bank_tile_cnt; i++ ) {

src/app/firedancer-dev/commands/repair.c

@@ -741,6 +741,7 @@ repair_cmd_fn_profiler_mode( args_t *   args,
 
   ulong turbine_slot0    = 0UL;
   long  last_print       = fd_log_wallclock();
+  ulong last_sent        = 0UL;
   fd_topo_run_single_process( &config->topo, 0, config->uid, config->gid, fdctl_tile_run );
   for(;;) {
 
@@ -771,7 +772,8 @@ repair_cmd_fn_profiler_mode( args_t *   args,
       printf( " | HighestWindow | %s |\n", fmt_count( buf2, repair_metrics[ MIDX( COUNTER, REPAIR, SENT_PKT_TYPES_NEEDED_HIGHEST_WINDOW ) ] ) );
       printf( " | Index         | %s |\n", fmt_count( buf2, repair_metrics[ MIDX( COUNTER, REPAIR, SENT_PKT_TYPES_NEEDED_WINDOW         ) ] ) );
       printf( " +---------------+--------------+\n" );
-      printf( " Send Pkt Rate: %s pps\n", fmt_count( buf2, repair_metrics[ MIDX( GAUGE, REPAIR, SEND_PKT_RATE ) ] ) );
+      printf( " Send Pkt Rate: %s pps\n",  fmt_count( buf2, (ulong)((sent - last_sent)*1e9L / (now - last_print) ) ) );
+      last_sent = sent;
 
       /* Sum overrun across all net tiles connected to repair_net */
       ulong total_overrun = repair_net_links[0][ MIDX( COUNTER, LINK, OVERRUN_POLLING_FRAG_COUNT ) ]; /* coarse double counting prevention */