feat: add log lines to dashboard

Author: starpit

plugins/plugin-codeflare-dashboard/src/components/Dashboard/Grid.tsx

@@ -17,8 +17,8 @@
 import React from "react"
 import { Box, BoxProps, Spacer, Text, TextProps } from "ink"
 
-import type { Props, State } from "./index.js"
-import type { UpdatePayload, Worker } from "./types.js"
+import type { Props } from "./index.js"
+import type { Worker } from "./types.js"
 
 import { avg } from "./stats.js"
 
@@ -30,7 +30,7 @@ type GridProps = {
   scale: Props["scale"]
   title: NonNullable<Props["grids"][number]>["title"]
   states: NonNullable<Props["grids"][number]>["states"]
-  workers: State["workers"][number]
+  workers: Worker[]
 }
 
 export default class Grid extends React.PureComponent<GridProps> {
@@ -74,7 +74,7 @@ export default class Grid extends React.PureComponent<GridProps> {
   }
 
   /** @return current `Worker[]` model */
-  private get workers(): UpdatePayload["workers"] {
+  private get workers(): Worker[] {
     return this.props.workers || []
   }
 

plugins/plugin-codeflare-dashboard/src/components/Dashboard/index.tsx

@@ -18,10 +18,11 @@ import React from "react"
 import prettyMillis from "pretty-ms"
 import { Box, Spacer, Text } from "ink"
 
-import type { GridSpec, UpdatePayload, Worker } from "./types.js"
+import type { GridSpec, UpdatePayload, LogLineUpdate, WorkersUpdate, Worker } from "./types.js"
 
 import Grid from "./Grid.js"
 import Timeline from "./Timeline.js"
+import { isWorkersUpdate } from "./types.js"
 
 export type Props = {
   /** CodeFlare Profile for this dashboard */
@@ -33,42 +34,39 @@ export type Props = {
   /** Scale up the grid? [default: 1] */
   scale?: number
 
+  /** Grid models, where null means to insert a line break */
   grids: (null | GridSpec)[]
 }
 
-export type State = {
-  /** millis since epoch of the first update */
-  firstUpdate: number
+export type State = Pick<WorkersUpdate, "events"> &
+  LogLineUpdate & {
+    /** millis since epoch of the first update */
+    firstUpdate: number
 
-  /** millis since epoch of the last update */
-  lastUpdate: number
+    /** millis since epoch of the last update */
+    lastUpdate: number
 
-  /** iteration count to help us keep "last updated ago" UI fresh */
-  iter: number
+    /** iteration count to help us keep "last updated ago" UI fresh */
+    iter: number
 
-  /** interval to keep "last updated ago" UI fresh */
-  agoInterval: ReturnType<typeof setInterval>
+    /** interval to keep "last updated ago" UI fresh */
+    agoInterval: ReturnType<typeof setInterval>
 
-  /** Lines of raw output to be displayed */
-  events: UpdatePayload["events"]
+    /** Controller that allows us to shut down gracefully */
+    watchers: { quit: () => void }[]
 
-  /** Controller that allows us to shut down gracefully */
-  watchers: { quit: () => void }[]
-
-  /**
-   * Model of current workers; outer idx is grid index; inner idx is
-   * worker idx, i.e. for each grid, we have an array of Workers.
-   */
-  workers: Worker[][]
-}
+    /**
+     * Model of current workers; outer idx is grid index; inner idx is
+     * worker idx, i.e. for each grid, we have an array of Workers.
+     */
+    workers: Worker[][]
+  }
 
 export default class Dashboard extends React.PureComponent<Props, State> {
   public componentDidMount() {
     this.setState({
       workers: [],
-      watchers: this.gridModels.map((props, gridIdx) =>
-        props.initWatcher((model: UpdatePayload) => this.onUpdate(gridIdx, model))
-      ),
+      watchers: this.gridModels.map((props, gridIdx) => props.initWatcher((model) => this.onUpdate(gridIdx, model))),
       agoInterval: setInterval(() => this.setState((curState) => ({ iter: (curState?.iter || 0) + 1 })), 5 * 1000),
     })
   }
@@ -87,8 +85,15 @@ export default class Dashboard extends React.PureComponent<Props, State> {
     this.setState((curState) => ({
       firstUpdate: (curState && curState.firstUpdate) || Date.now(), // TODO pull from the events
       lastUpdate: Date.now(), // TODO pull from the events
-      events: !model.events || model.events.length === 0 ? curState?.events : model.events,
-      workers: !curState?.workers
+      events: !isWorkersUpdate(model)
+        ? curState?.events
+        : !model.events || model.events.length === 0
+        ? curState?.events
+        : model.events,
+      logLine: !isWorkersUpdate(model) ? model.logLine : curState?.logLine,
+      workers: !isWorkersUpdate(model)
+        ? curState?.workers
+        : !curState?.workers
         ? [model.workers]
         : [...curState.workers.slice(0, gridIdx), model.workers, ...curState.workers.slice(gridIdx + 1)],
     }))
@@ -102,10 +107,15 @@ export default class Dashboard extends React.PureComponent<Props, State> {
   }
 
   /** @return current `events` model */
-  private get events(): UpdatePayload["events"] {
+  private get events(): State["events"] {
     return this.state?.events
   }
 
+  /** @return current `logLine` model */
+  private get logLine(): State["logLine"] {
+    return this.state?.logLine
+  }
+
   /** @return first update time */
   private get firstUpdate() {
     return this.state?.firstUpdate || Date.now()
@@ -184,17 +194,20 @@ export default class Dashboard extends React.PureComponent<Props, State> {
 
   /** Render log lines and events */
   private footer() {
-    if (!this.events) {
+    if (!this.events && !this.logLine) {
       return <React.Fragment />
     } else {
-      const rows = this.events.map(({ line, timestamp }) => {
-        // the controller (controller/dashboard/utilization/Live)
-        // leaves a {timestamp} breadcrumb in the raw line text, so
-        // that we,as the view, can inject a "5m ago" text, while
-        // preserving the ansi formatting that surrounds the timestamp
-        const txt = line.replace("{timestamp}", () => this.agos(timestamp))
-        return <Text key={txt}>{txt}</Text>
-      })
+      const rows = (this.events || [])
+        .map(({ line, timestamp }) => {
+          // the controller (controller/dashboard/utilization/Live)
+          // leaves a {timestamp} breadcrumb in the raw line text, so
+          // that we,as the view, can inject a "5m ago" text, while
+          // preserving the ansi formatting that surrounds the timestamp
+          const txt = line.replace("{timestamp}", () => this.agos(timestamp))
+          return <Text key={txt}>{txt}</Text>
+        })
+        .concat((this.logLine ? [this.logLine] : []).map((line) => <Text key={line}>{line}</Text>))
+
       return (
         <Box marginTop={1} flexDirection="column">
           {rows}

plugins/plugin-codeflare-dashboard/src/components/Dashboard/types.ts

@@ -37,15 +37,26 @@ export type Worker = {
   lastUpdate: number
 }
 
-/** Model that allows the controllers to pass updated `Worker` info */
-export type UpdatePayload = {
+export type LogLineUpdate = {
+  /** Log lines */
+  logLine: string
+}
+
+export type WorkersUpdate = {
   /** Per-worker status info */
   workers: Worker[]
 
   /** Lines of raw event lines to be displayed */
   events?: { line: string; timestamp: number }[]
 }
 
+/** Model that allows the controllers to pass updated `Worker` info */
+export type UpdatePayload = LogLineUpdate | WorkersUpdate
+
+export function isWorkersUpdate(update: UpdatePayload): update is WorkersUpdate {
+  return Array.isArray((update as WorkersUpdate).workers)
+}
+
 /** Callback from controller when it has updated data */
 export type OnData = (payload: UpdatePayload) => void
 

plugins/plugin-codeflare-dashboard/src/controller/dashboard/kinds.ts

@@ -16,23 +16,38 @@
 
 import type { SupportedGrid } from "./grids.js"
 
-type Kind = SupportedGrid | "logs"
+type Kind = SupportedGrid | "logs" | "env"
 export type KindA = Kind | "all"
 export default Kind
 
-export const resourcePaths: Record<Kind, string[]> = {
+/** A filepath with a `Kind` discriminant to help understand the content of the `filepath` */
+export type KindedSource = { kind: Kind; filepath: string }
+
+/**
+ * A source to be tailf'd is either a string (the filepath to the
+ * source) or that plus a `Kind` discriminant.
+ */
+type Source = string | KindedSource
+
+/** Extract the `filepath` property of `source` */
+export function filepathOf(source: Source) {
+  return typeof source === "string" ? source : source.filepath
+}
+
+export const resourcePaths: Record<Kind, Source[]> = {
   status: [
     "events/kubernetes.txt",
     "events/job-status.txt",
     "events/pods.txt",
     "events/runtime-env-setup.txt",
-    // "logs/job.txt",
+    { kind: "logs", filepath: "logs/job.txt" },
   ],
   "gpu%": ["resources/gpu.txt"],
   "gpumem%": ["resources/gpu.txt"],
   "cpu%": ["resources/pod-vmstat.txt"],
   "mem%": ["resources/pod-memory.txt"],
   logs: ["logs/job.txt"],
+  env: ["env.json"],
 }
 
 export function validKinds() {

plugins/plugin-codeflare-dashboard/src/controller/dashboard/options.ts

@@ -26,11 +26,14 @@ type Options = {
 
   /** Number of lines of events to show [default: 8] */
   events: number
+
+  /** Number of lines of application logs to show [default: 1] */
+  lines: number
 }
 
 export default Options
 
 export const flags = {
   boolean: ["demo"],
-  alias: { events: ["e"], theme: ["t"], demo: ["d"], scale: ["s"] },
+  alias: { events: ["e"], lines: ["l"], theme: ["t"], demo: ["d"], scale: ["s"] },
 }

plugins/plugin-codeflare-dashboard/src/controller/dashboard/status/Live.ts

@@ -40,6 +40,9 @@ export default class Live {
   /** Number of lines of event output to retain. TODO this depends on height of terminal? */
   private static readonly MAX_HEAP = 1000
 
+  /** Model of logLines. TODO circular buffer and obey options.lines */
+  // private logLine = ""
+
   /** Model of the lines of output */
   private readonly events = new Heap<Event>((a, b) => {
     if (a.line === b.line) {
@@ -62,9 +65,13 @@ export default class Live {
     private readonly opts: Pick<Options, "events">
   ) {
     tails.map((tailf) => {
-      tailf.then(({ stream }) => {
+      tailf.then(({ kind, stream }) => {
         stream.on("data", (data) => {
           if (data) {
+            if (kind === "logs") {
+              this.pushLineAndPublish(data, cb)
+            }
+
             const line = stripAnsi(data)
             const cols = line.split(/\s+/)
 
@@ -77,7 +84,6 @@ export default class Live {
 
             if (!name || !timestamp) {
               // console.error("Bad status record", line)
-              // this.pushEventAndPublish(data, metric, timestamp, cb)
               return
             } else if (!metric) {
               // ignoring this line
@@ -130,6 +136,14 @@ export default class Live {
     })
   }
 
+  /** @return the most important events, to be shown in the UI */
+  private importantEvents() {
+    return this.events
+      .toArray()
+      .slice(0, this.opts.events || 8)
+      .sort((a, b) => a.timestamp - b.timestamp)
+  }
+
   private readonly lookup: Record<string, Event> = {}
   /** Add `line` to our heap `this.events` */
   private pushEvent(line: string, metric: WorkerState, timestamp: number) {
@@ -163,16 +177,21 @@ export default class Live {
     if (this.opts.events === 0) {
       return []
     } else {
-      return this.events
-        .toArray()
-        .slice(0, this.opts.events || 8)
-        .sort((a, b) => a.timestamp - b.timestamp)
+      return this.importantEvents()
     }
   }
 
-  /** `pushEvent` and then pass the updated model to `cb` */
-  private pushEventAndPublish(line: string, metric: WorkerState, timestamp: number, cb: OnData) {
-    cb({ events: this.pushEvent(line, metric, timestamp), workers: Object.values(this.workers) })
+  /** Helps with debouncing logLine updates */
+  private logLineTO: null | ReturnType<typeof setTimeout> = null
+
+  /** Add the given `line` to our logLines model and pass the updated model to `cb` */
+  private pushLineAndPublish(logLine: string, cb: OnData) {
+    if (logLine) {
+      // here we avoid a flood of React renders by batching them up a
+      // bit; i thought react 18 was supposed to help with this. hmm.
+      if (this.logLineTO) clearTimeout(this.logLineTO)
+      this.logLineTO = setTimeout(() => cb({ logLine }), 1)
+    }
   }
 
   private asMillisSinceEpoch(timestamp: string) {

plugins/plugin-codeflare-dashboard/src/controller/dashboard/tailf.ts

@@ -19,9 +19,10 @@ import split2 from "split2"
 import chokidar from "chokidar"
 import TailFile from "@logdna/tail-file"
 
-import Kind, { resourcePaths } from "./kinds.js"
+import Kind, { KindedSource, resourcePaths } from "./kinds.js"
 
 export type Tail = {
+  kind: Kind
   stream: import("stream").Readable
   quit: TailFile["quit"]
 }
@@ -34,7 +35,7 @@ export function waitTillExists(filepath: string) {
   })
 }
 
-async function initTail(filepath: string, split = true): Promise<Tail> {
+async function initTail({ kind, filepath }: KindedSource, split = true): Promise<Tail> {
   await waitTillExists(filepath)
 
   return new Promise<Tail>((resolve, reject) => {
@@ -47,17 +48,23 @@ async function initTail(filepath: string, split = true): Promise<Tail> {
     tail.start()
 
     resolve({
+      kind,
       stream: split ? tail.pipe(split2()) : tail,
       quit: tail.quit.bind(tail),
     })
   })
 }
 
-export async function pathsFor(kind: Kind, profile: string, jobId: string) {
+export async function pathsFor(mkind: Kind, profile: string, jobId: string) {
   const { Profiles } = await import("madwizard")
-  return resourcePaths[kind].map((resourcePath) =>
-    join(Profiles.guidebookJobDataPath({ profile }), jobId, resourcePath)
-  )
+  return resourcePaths[mkind].map((src) => {
+    const kind = typeof src === "string" ? mkind : src.kind
+    const resourcePath = typeof src === "string" ? src : src.filepath
+    return {
+      kind,
+      filepath: join(Profiles.guidebookJobDataPath({ profile }), jobId, resourcePath),
+    }
+  })
 }
 
 export default async function tailf(
@@ -66,5 +73,5 @@ export default async function tailf(
   jobId: string,
   split = true
 ): Promise<Promise<Tail>[]> {
-  return pathsFor(kind, profile, jobId).then((_) => _.map((filepath) => initTail(filepath, split)))
+  return pathsFor(kind, profile, jobId).then((_) => _.map((src) => initTail(src, split)))
 }