chore: check server time to delete shred slots

Author: asuzuki-jumptrading

src/api/atoms.ts

@@ -30,6 +30,8 @@ import type {
   GossipPeersRowsUpdate,
   GossipPeersCellUpdate,
   ServerTimeNanos,
+  LiveNetworkMetrics,
+  TileMetrics,
 } from "./types";
 import { rafAtom } from "../atomUtils";
 
@@ -72,6 +74,10 @@ export const estimatedSlotDurationAtom = atom<
 
 export const estimatedTpsAtom = atom<EstimatedTps | undefined>(undefined);
 
+export const liveNetworkMetricsAtom = atom<LiveNetworkMetrics | undefined>(
+  undefined,
+);
+
 export const liveTxnWaterfallAtom = rafAtom<LiveTxnWaterfall | undefined>(
   undefined,
 );
@@ -80,6 +86,8 @@ export const liveTilePrimaryMetricAtom = atom<
   LiveTilePrimaryMetric | undefined
 >(undefined);
 
+export const liveTileMetricsAtom = atom<TileMetrics | undefined>(undefined);
+
 export const tileTimerAtom = atom<number[] | undefined>(undefined);
 
 export const bootProgressAtom = atom<BootProgress | undefined>(undefined);

src/api/consts.ts

@@ -1,4 +1,8 @@
 export const estimatedTpsDebounceMs = 400;
 export const liveMetricsDebounceMs = 100;
+export const liveTileMetricsDebounceMs = 130;
+export const liveNetworkMetricsDebounceMs = 130;
 export const waterfallDebounceMs = 100;
 export const tileTimerDebounceMs = 25;
+export const gossipNetworkDebounceMs = 300;
+export const gossipPeerSizeDebounceMs = 1_000;

src/api/entities.ts

@@ -128,6 +128,7 @@ export const resetSlotSchema = z.number().nullable();
 export const storageSlotSchema = z.number().nullable();
 export const voteSlotSchema = z.number();
 export const slotCaughtUpSchema = z.number().nullable();
+export const activeForkCountSchema = z.number();
 
 export const estimatedSlotDurationSchema = z.number();
 
@@ -503,6 +504,10 @@ export const summarySchema = z.discriminatedUnion("key", [
     key: z.literal("slot_caught_up"),
     value: slotCaughtUpSchema,
   }),
+  summaryTopicSchema.extend({
+    key: z.literal("active_fork_count"),
+    value: activeForkCountSchema,
+  }),
   summaryTopicSchema.extend({
     key: z.literal("estimated_slot_duration_nanos"),
     value: estimatedSlotDurationSchema,

src/api/types.ts

@@ -58,6 +58,8 @@ import type {
   gossipMessageStatsSchema,
   schedulerCountsSchema,
   serverTimeNanosSchema,
+  liveNetworkMetricsSchema,
+  tileMetricsSchema,
 } from "./entities";
 
 export type Client = z.infer<typeof clientSchema>;
@@ -105,12 +107,16 @@ export type EstimatedSlotDuration = z.infer<typeof estimatedSlotDurationSchema>;
 
 export type EstimatedTps = z.infer<typeof estimatedTpsSchema>;
 
+export type LiveNetworkMetrics = z.infer<typeof liveNetworkMetricsSchema>;
+
 export type LiveTxnWaterfall = z.infer<typeof liveTxnWaterfallSchema>;
 
 export type LiveTilePrimaryMetric = z.infer<typeof liveTilePrimaryMetricSchema>;
 
 export type TilePrimaryMetric = z.infer<typeof tilePrimaryMetricSchema>;
 
+export type TileMetrics = z.infer<typeof tileMetricsSchema>;
+
 export type TxnWaterfallIn = z.infer<typeof txnWaterfallInSchema>;
 export type TxnWaterfallOut = z.infer<typeof txnWaterfallOutSchema>;
 export type TxnWaterfall = z.infer<typeof txnWaterfallSchema>;

src/api/useSetAtomWsData.ts

@@ -28,6 +28,8 @@ import {
   gossipPeersRowsUpdateAtom,
   gossipPeersCellUpdateAtom,
   serverTimeNanosAtom,
+  liveNetworkMetricsAtom,
+  liveTileMetricsAtom,
 } from "./atoms";
 import {
   blockEngineSchema,
@@ -58,12 +60,14 @@ import type {
   EstimatedTps,
   GossipNetworkStats,
   GossipPeersSize,
+  LiveNetworkMetrics,
   LiveTilePrimaryMetric,
   LiveTxnWaterfall,
   Peer,
   PeerRemove,
   RepairSlot,
   SlotResponse,
+  TileMetrics,
   TurbineSlot,
 } from "./types";
 import { useDebouncedCallback, useThrottledCallback } from "use-debounce";
@@ -72,7 +76,11 @@ import { useServerMessages } from "./ws/utils";
 import { DateTime } from "luxon";
 import {
   estimatedTpsDebounceMs,
+  gossipNetworkDebounceMs,
+  gossipPeerSizeDebounceMs,
   liveMetricsDebounceMs,
+  liveNetworkMetricsDebounceMs,
+  liveTileMetricsDebounceMs,
   tileTimerDebounceMs,
   waterfallDebounceMs,
 } from "./consts";
@@ -133,6 +141,19 @@ export function useSetAtomWsData() {
     setEstimatedTps(value);
   }, estimatedTpsDebounceMs);
 
+  const setLiveNetworkMetrics = useSetAtom(liveNetworkMetricsAtom);
+  const setDbLiveNetworkMetrics = useThrottledCallback(
+    (value?: LiveNetworkMetrics) => {
+      setLiveNetworkMetrics(value);
+    },
+    liveNetworkMetricsDebounceMs,
+  );
+
+  const setLiveTileMetrics = useSetAtom(liveTileMetricsAtom);
+  const setDbLiveTileMetrics = useThrottledCallback((value?: TileMetrics) => {
+    setLiveTileMetrics(value);
+  }, liveTileMetricsDebounceMs);
+
   const setLivePrimaryMetrics = useSetAtom(liveTilePrimaryMetricAtom);
   const setDbLivePrimaryMetrics = useThrottledCallback(
     (value?: LiveTilePrimaryMetric) => {
@@ -178,15 +199,15 @@ export function useSetAtomWsData() {
     (value?: GossipNetworkStats) => {
       setGossipNetworkStats(value);
     },
-    300,
+    gossipNetworkDebounceMs,
   );
 
   const setGossipPeersSize = useSetAtom(gossipPeersSizeAtom);
   const setDbGossipPeersSize = useThrottledCallback(
     (value?: GossipPeersSize) => {
       setGossipPeersSize(value);
     },
-    1_000,
+    gossipPeerSizeDebounceMs,
   );
   const setGossipPeersRows = useSetAtom(gossipPeersRowsUpdateAtom);
   const setGossipPeersCells = useSetAtom(gossipPeersCellUpdateAtom);
@@ -389,6 +410,13 @@ export function useSetAtomWsData() {
             setServerTimeNanos(value);
             break;
           }
+          case "live_network_metrics": {
+            setDbLiveNetworkMetrics(value);
+            break;
+          }
+          case "live_tile_metrics":
+            setDbLiveTileMetrics(value);
+            break;
           case "root_slot":
           case "optimistically_confirmed_slot":
           case "estimated_slot":
@@ -398,8 +426,7 @@ export function useSetAtomWsData() {
           case "storage_slot":
           case "vote_slot":
           case "slot_caught_up":
-          case "live_network_metrics":
-          case "live_tile_metrics":
+          case "active_fork_count":
             break;
         }
       } else if (topic === "epoch") {

src/clockUtils.ts

@@ -1,14 +1,19 @@
 type Sub = (now: number, dt: number) => void;
 
-export function clockSub(intervalMs: number) {
+export function clockSub(_intervalMs: number) {
   const subs = new Set<Sub>();
   let id: number | null = null;
   let last = performance.now();
+  let intervalMs = _intervalMs;
 
-  function startChartClock() {
+  function startChartClock(newIntervalMs?: number) {
     if (id == null) {
       stopChartClock();
     }
+    if (newIntervalMs !== undefined) {
+      stopChartClock();
+      intervalMs = newIntervalMs;
+    }
 
     const loop = () => {
       const now = performance.now();
@@ -37,5 +42,5 @@ export function clockSub(intervalMs: number) {
 
   startChartClock();
 
-  return { subscribeClock, stopChartClock };
+  return { subscribeClock, stopChartClock, startChartClock };
 }

src/colors.ts

@@ -283,3 +283,6 @@ export const slotsListFutureSlotColor = "#878787";
 export const slotsListCurrentSlotBoxShadowColor = "rgba(191, 135, 253, 0.13)";
 export const slotsListCurrentSlotNumberBackgroundColor = "#283551";
 export const slotsListNextLeaderProgressBarColor = "#37a4bc";
+
+// Tile charts
+export const tileChartDarkBackground = "#0000001F";

src/features/Overview/LiveNetworkMetrics/consts.ts

@@ -0,0 +1,7 @@
+export const networkProtocols = [
+  "turbine",
+  "gossip",
+  "tpu",
+  "repair",
+  "metrics",
+];

src/features/Overview/LiveNetworkMetrics/index.tsx

@@ -0,0 +1,137 @@
+import { useAtomValue } from "jotai";
+import { liveNetworkMetricsAtom } from "../../../api/atoms";
+import Card from "../../../components/Card";
+import { Flex, Table, Text } from "@radix-ui/themes";
+import tableStyles from "../../Gossip/table.module.css";
+import { useEmaValue } from "../../../hooks/useEma";
+import { networkProtocols } from "./consts";
+import { formatBytesAsBits } from "../../../utils";
+import { Bars } from "../../StartupProgress/Firedancer/Bars";
+import TileSparkLine from "../SlotPerformance/TileSparkLine";
+import { headerGap } from "../../Gossip/consts";
+import type { CSSProperties } from "react";
+import styles from "./liveNetworkMetrics.module.css";
+import { sum } from "lodash";
+import { tileChartDarkBackground } from "../../../colors";
+
+const chartHeight = 18;
+
+export default function LiveNetworkMetrics() {
+  const liveNetworkMetrics = useAtomValue(liveNetworkMetricsAtom);
+  if (!liveNetworkMetrics) return;
+
+  return (
+    <Flex wrap="wrap" gap="4">
+      <NetworkMetricsCard metrics={liveNetworkMetrics.ingress} type="Ingress" />
+      <NetworkMetricsCard metrics={liveNetworkMetrics.egress} type="Egress" />
+    </Flex>
+  );
+}
+
+interface NetworkMetricsCardProps {
+  metrics: number[];
+  type: "Ingress" | "Egress";
+}
+
+function NetworkMetricsCard({ metrics, type }: NetworkMetricsCardProps) {
+  return (
+    <Card style={{ flexGrow: 1 }}>
+      <Flex direction="column" height="100%" gap={headerGap}>
+        <Text className={tableStyles.headerText}>Network {type}</Text>
+        <Table.Root
+          variant="ghost"
+          className={tableStyles.root}
+          size="1"
+          style={{ "--bar-height": `${chartHeight}px` } as CSSProperties}
+        >
+          <Table.Header>
+            <Table.Row>
+              <Table.ColumnHeaderCell width="60px">
+                Protocol
+              </Table.ColumnHeaderCell>
+              <Table.ColumnHeaderCell align="right" width="80px">
+                Current
+              </Table.ColumnHeaderCell>
+              <Table.ColumnHeaderCell
+                minWidth={{
+                  xl: "250px",
+                  lg: "160px",
+                  md: "100px",
+                  initial: "60px",
+                }}
+              >
+                Utilization
+              </Table.ColumnHeaderCell>
+              <Table.ColumnHeaderCell
+                align="right"
+                width={{
+                  xl: "240px",
+                  lg: "200px",
+                  md: "100px",
+                  initial: "200px",
+                }}
+              >
+                History (1m)
+              </Table.ColumnHeaderCell>
+            </Table.Row>
+          </Table.Header>
+
+          <Table.Body>
+            {metrics.map((value, i) => (
+              <TableRow key={i} value={value} idx={i} />
+            ))}
+            <TableRow
+              value={sum(metrics)}
+              label="Total"
+              className={styles.totalRow}
+            />
+          </Table.Body>
+        </Table.Root>
+      </Flex>
+    </Card>
+  );
+}
+
+const maxValue = 100_000_000;
+
+interface TableRowProps {
+  value: number;
+  idx?: number;
+  label?: string;
+}
+
+function TableRow({
+  value,
+  idx,
+  label,
+  ...props
+}: TableRowProps & Table.RootProps) {
+  const emaValue = useEmaValue(value);
+  const formattedValue = formatBytesAsBits(emaValue);
+
+  return (
+    <Table.Row {...props}>
+      <Table.RowHeaderCell>
+        {label ?? networkProtocols[idx ?? -1]}
+      </Table.RowHeaderCell>
+      <Table.Cell align="right">
+        {formattedValue.value} {formattedValue.unit}
+      </Table.Cell>
+      <Table.Cell className={styles.chart}>
+        <Flex align="center">
+          <Bars value={emaValue} max={maxValue} barWidth={2} />
+        </Flex>
+      </Table.Cell>
+      <Table.Cell className={styles.chart}>
+        <TileSparkLine
+          value={Math.min(1, emaValue / maxValue)}
+          background={tileChartDarkBackground}
+          windowMs={60_000}
+          height={chartHeight}
+          updateIntervalMs={500}
+          tickMs={1_000}
+        />
+      </Table.Cell>
+    </Table.Row>
+  );
+}

src/features/Overview/LiveNetworkMetrics/liveNetworkMetrics.module.css

@@ -0,0 +1,5 @@
+.chart {
+  padding-top: 0;
+  padding-bottom: 0;
+  vertical-align: middle;
+}