feat: shreds progression

Author: asuzuki-jumptrading

package-lock.json

@@ -68,16 +68,19 @@
     "@eslint/js": "^9.11.1",
     "@tanstack/router-devtools": "^1.114.25",
     "@tanstack/router-vite-plugin": "^1.114.25",
+    "@testing-library/dom": "^10.4.1",
+    "@testing-library/react": "^16.3.0",
     "@types/node": "^22.7.0",
-    "@types/react": "^18.3.9",
-    "@types/react-dom": "^18.3.0",
+    "@types/react": "^18.3.26",
+    "@types/react-dom": "^18.3.7",
     "@vitejs/plugin-react": "^4.3.1",
     "eslint": "^9.11.1",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-react": "^7.36.1",
     "eslint-plugin-react-hooks": "^5.1.0-rc-04bd67a4-20240924",
     "eslint-plugin-react-refresh": "^0.4.12",
     "husky": "^9.1.7",
+    "jsdom": "^27.1.0",
     "lint-staged": "^15.5.0",
     "optionator": "^0.9.4",
     "prettier": "^3.5.3",
@@ -87,9 +90,9 @@
     "typescript-plugin-css-modules": "^5.1.0",
     "vite": "^5.4.8",
     "vite-plugin-checker": "^0.8.0",
+    "vite-plugin-svgr": "^4.5.0",
     "vite-plugin-top-level-await": "^1.5.0",
     "vite-plugin-wasm": "^3.4.1",
-    "vite-plugin-svgr": "^4.5.0",
     "vitest": "^3.0.9"
   }
 }

package.json

@@ -651,6 +651,7 @@ export const slotResponseSchema = z.object({
 });
 
 export const slotSkippedHistorySchema = z.number().array();
+export const slotSkippedHistoryClusterSchema = z.number().array();
 
 export const slotRankingsSchema = z.object({
   slots_largest_tips: z.number().array(),
@@ -679,11 +680,32 @@ export const slotRankingsSchema = z.object({
   vals_smallest_skipped: z.coerce.bigint().array(),
 });
 
+export enum ShredEvent {
+  shred_repair_request = 0,
+  shred_received_turbine,
+  shred_received_repair,
+  shred_replayed,
+  slot_complete,
+}
+
+export const liveShredsSchema = z.object({
+  reference_slot: z.number(),
+  reference_ts: z.coerce.bigint(),
+  slot_delta: z.number().array(),
+  shred_idx: z.number().nullable().array(),
+  event: z.nativeEnum(ShredEvent).array(),
+  event_ts_delta: z.coerce.number().array(),
+});
+
 export const slotSchema = z.discriminatedUnion("key", [
   slotTopicSchema.extend({
     key: z.literal("skipped_history"),
     value: slotSkippedHistorySchema,
   }),
+  slotTopicSchema.extend({
+    key: z.literal("skipped_history_cluster"),
+    value: slotSkippedHistoryClusterSchema,
+  }),
   slotTopicSchema.extend({
     key: z.literal("update"),
     value: slotResponseSchema,
@@ -696,6 +718,10 @@ export const slotSchema = z.discriminatedUnion("key", [
     key: z.literal("query_rankings"),
     value: slotRankingsSchema,
   }),
+  slotTopicSchema.extend({
+    key: z.literal("live_shreds"),
+    value: liveShredsSchema,
+  }),
 ]);
 
 export const blockEngineStatusSchema = z.enum([

src/api/entities.ts

@@ -47,6 +47,7 @@ import type {
   catchUpHistorySchema,
   repairSlotSchema,
   turbineSlotSchema,
+  liveShredsSchema,
 } from "./entities";
 
 export type Client = z.infer<typeof clientSchema>;
@@ -150,3 +151,4 @@ export type BlockEngineUpdate = z.infer<typeof blockEngineUpdateSchema>;
 export type BlockEngineStatus = z.infer<typeof blockEngineStatusSchema>;
 
 export type SlotRankings = z.infer<typeof slotRankingsSchema>;
+export type LiveShreds = z.infer<typeof liveShredsSchema>;

src/api/types.ts

@@ -45,6 +45,9 @@ import {
   deleteSlotStatusBoundsAtom,
   deleteSlotResponseBoundsAtom,
   skipRateAtom,
+  addSkippedClusterSlotsAtom,
+  deleteSkippedClusterSlotAtom,
+  deleteSkippedClusterSlotsRangeAtom,
 } from "../atoms";
 import type {
   EstimatedSlotDuration,
@@ -71,6 +74,11 @@ import {
   addTurbineSlotsAtom,
   addRepairSlotsAtom,
 } from "../features/StartupProgress/Firedancer/CatchingUp/atoms";
+import {
+  addLiveShredsAtom,
+  deleteLiveShredsAtom,
+} from "../features/Overview/ShredsProgression/atoms";
+import { xRangeMs } from "../features/Overview/ShredsProgression/const";
 
 export function useSetAtomWsData() {
   const setVersion = useSetAtom(versionAtom);
@@ -158,9 +166,18 @@ export function useSetAtomWsData() {
 
   const setCompletedSlot = useSetAtom(completedSlotAtom);
 
+  const addSkippedClusterSlots = useSetAtom(addSkippedClusterSlotsAtom);
+  const deleteSkippedClusterSlot = useSetAtom(deleteSkippedClusterSlotAtom);
+
   const handleSlotUpdate = (value: SlotResponse) => {
     setSlotStatus(value.publish.slot, value.publish.level);
 
+    if (value.publish.skipped) {
+      addSkippedClusterSlots([value.publish.slot]);
+    } else {
+      deleteSkippedClusterSlot(value.publish.slot);
+    }
+
     if (value.publish.mine) {
       if (value.publish.skipped) {
         setSkippedSlots((prev) =>
@@ -193,6 +210,8 @@ export function useSetAtomWsData() {
     addRepairSlots([slot]);
   };
 
+  const addLiveShreds = useSetAtom(addLiveShredsAtom);
+
   useServerMessages((msg) => {
     try {
       const { topic } = topicSchema.parse(msg);
@@ -329,6 +348,10 @@ export function useSetAtomWsData() {
             setSkippedSlots(value.sort());
             break;
           }
+          case "skipped_history_cluster": {
+            addSkippedClusterSlots(value);
+            break;
+          }
           case "update":
           case "query": {
             if (value) {
@@ -341,6 +364,10 @@ export function useSetAtomWsData() {
             setSlotRankings(value);
             break;
           }
+          case "live_shreds": {
+            addLiveShreds(value);
+            break;
+          }
         }
       } else if (topic === "block_engine") {
         const { key, value } = blockEngineSchema.parse(msg);
@@ -375,6 +402,9 @@ export function useSetAtomWsData() {
 
   const deleteSlotStatusBounds = useSetAtom(deleteSlotStatusBoundsAtom);
   const deleteSlotResponseBounds = useSetAtom(deleteSlotResponseBoundsAtom);
+  const deleteSkippedClusterSlotsRange = useSetAtom(
+    deleteSkippedClusterSlotsRangeAtom,
+  );
 
   useInterval(() => {
     deleteSlotStatusBounds();
@@ -386,6 +416,14 @@ export function useSetAtomWsData() {
           (slot) => slot >= epoch.start_slot && slot <= epoch.end_slot,
         );
       });
+
+      deleteSkippedClusterSlotsRange(epoch.start_slot, epoch.end_slot);
     }
   }, 5_000);
+
+  const deleteLiveShreds = useSetAtom(deleteLiveShredsAtom);
+
+  useInterval(() => {
+    deleteLiveShreds();
+  }, xRangeMs / 4);
 }

src/api/useSetAtomWsData.ts

@@ -659,3 +659,42 @@ export const statusAtom = atom<Status | null>((get) => {
 
   return "Past";
 });
+
+export const [
+  skippedClusterSlotsAtom,
+  addSkippedClusterSlotsAtom,
+  deleteSkippedClusterSlotAtom,
+  deleteSkippedClusterSlotsRangeAtom,
+] = (function getSkippedSlotsClusterAtom() {
+  const _skippedClusterSlotsAtom = atomWithImmer(new Set<number>());
+  return [
+    atom((get) => get(_skippedClusterSlotsAtom)),
+    atom(null, (_get, set, slots: number[]) => {
+      set(_skippedClusterSlotsAtom, (prev) => {
+        for (const slot of slots) {
+          prev.add(slot);
+        }
+        return prev;
+      });
+    }),
+    atom(null, (_get, set, slot: number) => {
+      set(_skippedClusterSlotsAtom, (prev) => {
+        prev.delete(slot);
+        return prev;
+      });
+    }),
+
+    atom(null, (_get, set, startSlot: number, endSlot: number) => {
+      set(_skippedClusterSlotsAtom, (prev) => {
+        const toKeep = new Set<number>();
+
+        for (const slot of prev) {
+          if (slot < startSlot || slot > endSlot) continue;
+          toKeep.add(slot);
+        }
+
+        return toKeep;
+      });
+    }),
+  ];
+})();

src/atoms.ts

@@ -136,6 +136,16 @@ export const tilePrimaryStatValueColor = "#DB8F38";
 
 export const nextSlotValueColor = "#CACACA";
 
+// shreds
+export const gridTicksColor = "#858585";
+export const gridLineColor = "#312D42";
+
+export const shredRepairRequestedColor = "#792C2C";
+export const shredReceivedTurbineColor = "#163454";
+export const shredReceivedRepairColor = "#89603E";
+export const shredReplayedColor = "#408E7B";
+export const shredSkippedColor = "#FF5353";
+
 // epoch bar
 export const epochTextColor = "#FAFAFA";
 export const epochNotLiveColor = "#3CB4FF";