feat: new overview tile and network stats

Author: atsai-jumptrading

src/api/atoms.ts

@@ -29,6 +29,8 @@ import type {
   GossipPeersSize,
   GossipPeersRowsUpdate,
   GossipPeersCellUpdate,
+  LiveNetworkMetrics,
+  TileMetrics,
 } from "./types";
 import { rafAtom } from "../atomUtils";
 
@@ -70,6 +72,10 @@ export const estimatedSlotDurationAtom = atom<
 
 export const estimatedTpsAtom = atom<EstimatedTps | undefined>(undefined);
 
+export const liveNetworkMetricsAtom = atom<LiveNetworkMetrics | undefined>(
+  undefined,
+);
+
 export const liveTxnWaterfallAtom = rafAtom<LiveTxnWaterfall | undefined>(
   undefined,
 );
@@ -78,6 +84,8 @@ export const liveTilePrimaryMetricAtom = atom<
   LiveTilePrimaryMetric | undefined
 >(undefined);
 
+export const liveTileMetricsAtom = atom<TileMetrics | undefined>(undefined);
+
 export const tileTimerAtom = atom<number[] | undefined>(undefined);
 
 export const bootProgressAtom = atom<BootProgress | undefined>(undefined);

src/api/consts.ts

@@ -1,4 +1,8 @@
 export const estimatedTpsDebounceMs = 400;
 export const liveMetricsDebounceMs = 100;
+export const liveTileMetricsDebounceMs = 130;
+export const liveNetworkMetricsDebounceMs = 130;
 export const waterfallDebounceMs = 100;
 export const tileTimerDebounceMs = 25;
+export const gossipNetworkDebounceMs = 300;
+export const gossipPeerSizeDebounceMs = 1_000;

src/api/entities.ts

@@ -126,6 +126,7 @@ export const resetSlotSchema = z.number();
 export const storageSlotSchema = z.number();
 export const voteSlotSchema = z.number();
 export const slotCaughtUpSchema = z.number().nullable();
+export const activeForkCountSchema = z.number();
 
 export const estimatedSlotDurationSchema = z.number();
 
@@ -501,6 +502,10 @@ export const summarySchema = z.discriminatedUnion("key", [
     key: z.literal("slot_caught_up"),
     value: slotCaughtUpSchema,
   }),
+  summaryTopicSchema.extend({
+    key: z.literal("active_fork_count"),
+    value: activeForkCountSchema,
+  }),
   summaryTopicSchema.extend({
     key: z.literal("estimated_slot_duration_nanos"),
     value: estimatedSlotDurationSchema,

src/api/types.ts

@@ -57,6 +57,8 @@ import type {
   gossipStorageStatsSchema,
   gossipMessageStatsSchema,
   schedulerCountsSchema,
+  liveNetworkMetricsSchema,
+  tileMetricsSchema,
 } from "./entities";
 
 export type Client = z.infer<typeof clientSchema>;
@@ -102,12 +104,16 @@ export type EstimatedSlotDuration = z.infer<typeof estimatedSlotDurationSchema>;
 
 export type EstimatedTps = z.infer<typeof estimatedTpsSchema>;
 
+export type LiveNetworkMetrics = z.infer<typeof liveNetworkMetricsSchema>;
+
 export type LiveTxnWaterfall = z.infer<typeof liveTxnWaterfallSchema>;
 
 export type LiveTilePrimaryMetric = z.infer<typeof liveTilePrimaryMetricSchema>;
 
 export type TilePrimaryMetric = z.infer<typeof tilePrimaryMetricSchema>;
 
+export type TileMetrics = z.infer<typeof tileMetricsSchema>;
+
 export type TxnWaterfallIn = z.infer<typeof txnWaterfallInSchema>;
 export type TxnWaterfallOut = z.infer<typeof txnWaterfallOutSchema>;
 export type TxnWaterfall = z.infer<typeof txnWaterfallSchema>;

src/api/useSetAtomWsData.ts

@@ -27,6 +27,8 @@ import {
   gossipPeersSizeAtom,
   gossipPeersRowsUpdateAtom,
   gossipPeersCellUpdateAtom,
+  liveNetworkMetricsAtom,
+  liveTileMetricsAtom,
 } from "./atoms";
 import {
   blockEngineSchema,
@@ -57,12 +59,14 @@ import type {
   EstimatedTps,
   GossipNetworkStats,
   GossipPeersSize,
+  LiveNetworkMetrics,
   LiveTilePrimaryMetric,
   LiveTxnWaterfall,
   Peer,
   PeerRemove,
   RepairSlot,
   SlotResponse,
+  TileMetrics,
   TurbineSlot,
 } from "./types";
 import { useDebouncedCallback, useThrottledCallback } from "use-debounce";
@@ -71,7 +75,11 @@ import { useServerMessages } from "./ws/utils";
 import { DateTime } from "luxon";
 import {
   estimatedTpsDebounceMs,
+  gossipNetworkDebounceMs,
+  gossipPeerSizeDebounceMs,
   liveMetricsDebounceMs,
+  liveNetworkMetricsDebounceMs,
+  liveTileMetricsDebounceMs,
   tileTimerDebounceMs,
   waterfallDebounceMs,
 } from "./consts";
@@ -132,6 +140,19 @@ export function useSetAtomWsData() {
     setEstimatedTps(value);
   }, estimatedTpsDebounceMs);
 
+  const setLiveNetworkMetrics = useSetAtom(liveNetworkMetricsAtom);
+  const setDbLiveNetworkMetrics = useThrottledCallback(
+    (value?: LiveNetworkMetrics) => {
+      setLiveNetworkMetrics(value);
+    },
+    liveNetworkMetricsDebounceMs,
+  );
+
+  const setLiveTileMetrics = useSetAtom(liveTileMetricsAtom);
+  const setDbLiveTileMetrics = useThrottledCallback((value?: TileMetrics) => {
+    setLiveTileMetrics(value);
+  }, liveTileMetricsDebounceMs);
+
   const setLivePrimaryMetrics = useSetAtom(liveTilePrimaryMetricAtom);
   const setDbLivePrimaryMetrics = useThrottledCallback(
     (value?: LiveTilePrimaryMetric) => {
@@ -177,15 +198,15 @@ export function useSetAtomWsData() {
     (value?: GossipNetworkStats) => {
       setGossipNetworkStats(value);
     },
-    300,
+    gossipNetworkDebounceMs,
   );
 
   const setGossipPeersSize = useSetAtom(gossipPeersSizeAtom);
   const setDbGossipPeersSize = useThrottledCallback(
     (value?: GossipPeersSize) => {
       setGossipPeersSize(value);
     },
-    1_000,
+    gossipPeerSizeDebounceMs,
   );
   const setGossipPeersRows = useSetAtom(gossipPeersRowsUpdateAtom);
   const setGossipPeersCells = useSetAtom(gossipPeersCellUpdateAtom);
@@ -383,6 +404,13 @@ export function useSetAtomWsData() {
             addRepairSlots(value.repair);
             break;
           }
+          case "live_network_metrics": {
+            setDbLiveNetworkMetrics(value);
+            break;
+          }
+          case "live_tile_metrics":
+            setDbLiveTileMetrics(value);
+            break;
           case "root_slot":
           case "optimistically_confirmed_slot":
           case "estimated_slot":
@@ -392,8 +420,7 @@ export function useSetAtomWsData() {
           case "storage_slot":
           case "vote_slot":
           case "slot_caught_up":
-          case "live_network_metrics":
-          case "live_tile_metrics":
+          case "active_fork_count":
             break;
         }
       } else if (topic === "epoch") {

src/clockUtils.ts

@@ -1,14 +1,19 @@
 type Sub = (now: number, dt: number) => void;
 
-export function clockSub(intervalMs: number) {
+export function clockSub(_intervalMs: number) {
   const subs = new Set<Sub>();
   let id: number | null = null;
   let last = performance.now();
+  let intervalMs = _intervalMs;
 
-  function startChartClock() {
+  function startChartClock(newIntervalMs?: number) {
     if (id == null) {
       stopChartClock();
     }
+    if (newIntervalMs !== undefined) {
+      stopChartClock();
+      intervalMs = newIntervalMs;
+    }
 
     const loop = () => {
       const now = performance.now();
@@ -37,5 +42,5 @@ export function clockSub(intervalMs: number) {
 
   startChartClock();
 
-  return { subscribeClock, stopChartClock };
+  return { subscribeClock, stopChartClock, startChartClock };
 }

src/features/Overview/LiveNetworkMetrics/consts.ts

@@ -0,0 +1,7 @@
+export const networkProtocols = [
+  "turbine",
+  "gossip",
+  "tpu",
+  "repair",
+  "metrics",
+];

src/features/Overview/LiveNetworkMetrics/index.tsx

@@ -0,0 +1,136 @@
+import { useAtomValue } from "jotai";
+import { liveNetworkMetricsAtom } from "../../../api/atoms";
+import Card from "../../../components/Card";
+import { Flex, Table, Text } from "@radix-ui/themes";
+import tableStyles from "../../Gossip/table.module.css";
+import { useEmaValue } from "../../../hooks/useEma";
+import { networkProtocols } from "./consts";
+import { formatBytesAsBits } from "../../../utils";
+import { Bars } from "../../StartupProgress/Firedancer/Bars";
+import TileSparkLine from "../SlotPerformance/TileSparkLine";
+import { headerGap } from "../../Gossip/consts";
+import type { CSSProperties } from "react";
+import styles from "./liveNetworkMetrics.module.css";
+import { sum } from "lodash";
+
+const chartHeight = 18;
+
+export default function LiveNetworkMetrics() {
+  const liveNetworkMetrics = useAtomValue(liveNetworkMetricsAtom);
+  if (!liveNetworkMetrics) return;
+
+  return (
+    <Flex wrap="wrap" gap="4">
+      <NetworkMetricsCard metrics={liveNetworkMetrics.ingress} type="Ingress" />
+      <NetworkMetricsCard metrics={liveNetworkMetrics.egress} type="Egress" />
+    </Flex>
+  );
+}
+
+interface NetworkMetricsCardProps {
+  metrics: number[];
+  type: "Ingress" | "Egress";
+}
+
+function NetworkMetricsCard({ metrics, type }: NetworkMetricsCardProps) {
+  return (
+    <Card style={{ flexGrow: 1 }}>
+      <Flex direction="column" height="100%" gap={headerGap}>
+        <Text className={tableStyles.headerText}>Network {type}</Text>
+        <Table.Root
+          variant="ghost"
+          className={tableStyles.root}
+          size="1"
+          style={{ "--bar-height": `${chartHeight}px` } as CSSProperties}
+        >
+          <Table.Header>
+            <Table.Row>
+              <Table.ColumnHeaderCell width="60px">
+                Protocol
+              </Table.ColumnHeaderCell>
+              <Table.ColumnHeaderCell align="right" width="80px">
+                Current
+              </Table.ColumnHeaderCell>
+              <Table.ColumnHeaderCell
+                minWidth={{
+                  xl: "250px",
+                  lg: "160px",
+                  md: "100px",
+                  initial: "60px",
+                }}
+              >
+                Utilization
+              </Table.ColumnHeaderCell>
+              <Table.ColumnHeaderCell
+                align="right"
+                width={{
+                  xl: "240px",
+                  lg: "200px",
+                  md: "100px",
+                  initial: "200px",
+                }}
+              >
+                History (1m)
+              </Table.ColumnHeaderCell>
+            </Table.Row>
+          </Table.Header>
+
+          <Table.Body>
+            {metrics.map((value, i) => (
+              <TableRow key={i} value={value} idx={i} />
+            ))}
+            <TableRow
+              value={sum(metrics)}
+              label="Total"
+              className={styles.totalRow}
+            />
+          </Table.Body>
+        </Table.Root>
+      </Flex>
+    </Card>
+  );
+}
+
+const maxValue = 100_000_000;
+
+interface TableRowProps {
+  value: number;
+  idx?: number;
+  label?: string;
+}
+
+function TableRow({
+  value,
+  idx,
+  label,
+  ...props
+}: TableRowProps & Table.RootProps) {
+  const emaValue = useEmaValue(value);
+  const formattedValue = formatBytesAsBits(emaValue);
+
+  return (
+    <Table.Row {...props}>
+      <Table.RowHeaderCell>
+        {label ?? networkProtocols[idx ?? -1]}
+      </Table.RowHeaderCell>
+      <Table.Cell align="right">
+        {formattedValue.value} {formattedValue.unit}
+      </Table.Cell>
+      <Table.Cell className={styles.chart}>
+        <Flex align="center">
+          <Bars value={emaValue} max={maxValue} barWidth={2} />
+        </Flex>
+      </Table.Cell>
+      <Table.Cell className={styles.chart}>
+        <TileSparkLine
+          value={Math.min(1, emaValue / maxValue)}
+          includeBg={false}
+          windowMs={60_000}
+          height={chartHeight}
+          updateIntervalMs={500}
+          tickMs={1_000}
+        />
+      </Table.Cell>
+    </Table.Row>
+  );
+}

src/features/Overview/LiveNetworkMetrics/liveNetworkMetrics.module.css

@@ -0,0 +1,5 @@
+.chart {
+  padding-top: 0;
+  padding-bottom: 0;
+  vertical-align: middle;
+}

src/features/Overview/LiveTileMetrics/consts.ts

@@ -0,0 +1,22 @@
+/** Tile regimes are the cartesian product of the following two state vectors:
+    State vector 1:
+    running: means that at the time the run loop executed, there was no upstream message I/O for the tile to handle.
+    processing: means that at the time the run loop executed, there was one or more messages for the tile to consume.
+    stalled: means that at the time the run loop executed, a downstream consumer of the messages produced by this tile is slow or stalled, and the message link for that consumer has filled up. This state causes the tile to stop processing upstream messages.
+    
+    State Vector 2:
+    maintenance: the portion of the run loop that executes infrequent, potentially CPU heavy tasks
+    routine: the portion of the run loop that executes regularly, regardless of the presence of incoming messages
+    handling: the portion of the run loop that executes as a side effect of an incoming message from an upstream producer tile
+ */
+export const regimes = [
+  "running_maintenance",
+  "processing_maintenance",
+  "stalled_maintenance",
+  "running_routine",
+  "processing_routine",
+  "stalled_routine",
+  "running_handling",
+  "processing_handling",
+  // "stalled_handling" is an impossible state, and is therefore excluded
+];