fix: add gpu stats to dashboard

this also adjusts the utilization colors so that lower utilizations get less saturated colors, and higher utilizations get more saturated colors

Author: starpit

package-lock.json

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 The Kubernetes Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type Kind from "./kinds.js"
+import type { ValidTheme as ValidUtilizationTheme } from "./themes/utilization.js"
+
+export type SupportedUtilizationGrid = "cpu%" | "mem%" | "gpu%" | "gpumem%"
+
+export type SupportedGrid = "status" | SupportedUtilizationGrid
+
+export const defaultUtilizationThemes: Record<SupportedUtilizationGrid, ValidUtilizationTheme> = {
+  "cpu%": "rain",
+  "mem%": "purple",
+  "gpu%": "red",
+  "gpumem%": "magenta",
+}
+
+export const providerFor: Record<SupportedUtilizationGrid, string> = {
+  "cpu%": "CPU Utilization",
+  "mem%": "Mem Utilization",
+  "gpu%": "GPU Utilization",
+  "gpumem%": "Mem Utilization",
+}
+
+export function isSupportedGrid(kind: Kind): kind is SupportedGrid {
+  return kind === "status" || Object.keys(defaultUtilizationThemes).includes(kind)
+}

plugins/plugin-codeflare-dashboard/src/controller/dashboard/grids.ts

@@ -20,6 +20,7 @@ import tailf from "./tailf.js"
 import dashboardUI from "./db.js"
 import status from "./status.js"
 import utilization from "./utilization.js"
+import { SupportedGrid, isSupportedGrid } from "./grids.js"
 import { KindA, isValidKindA, validKinds } from "./kinds.js"
 import type { GridSpec } from "../../components/Dashboard/index.js"
 
@@ -39,8 +40,8 @@ function enterAltBufferMode() {
   console.log("\x1b[?1049h")
 }
 
-export function usage(cmd: string) {
-  return `Usage: codeflare ${cmd} ${validKinds().join("|")} [<jobId>|-N]`
+export function usage(cmd: string, extraKinds: string[] = []) {
+  return `Usage: codeflare ${cmd} ${extraKinds.concat(validKinds()).join("|")} [<jobId>|-N]`
 }
 
 async function lastNJob(profile: string, N: number) {
@@ -90,25 +91,29 @@ export default async function dashboard(args: Arguments<Options>, cmd: "db" | "d
   const { jobId, profile } = await jobIdFrom(args, cmd)
 
   if (!isValidKindA(kind)) {
-    throw new Error(usage(cmd))
+    throw new Error(usage(cmd, ["all"]))
   } else if (!jobId) {
-    throw new Error(usage(cmd))
+    throw new Error(usage(cmd, ["all"]))
   }
 
-  const gridFor = async (kind: "status" | "cpu" | "memory"): Promise<GridSpec> => {
+  const gridFor = async (kind: SupportedGrid): Promise<GridSpec> => {
     const tails = await tailf(kind, profile, jobId)
     return kind === "status"
       ? status(tails, { demo, theme, themeDefault: "patternfly" })
-      : kind === "cpu"
-      ? utilization(kind, tails, { demo, theme, themeDefault: "rain" })
-      : utilization(kind, tails, { demo, theme, themeDefault: "purple" })
+      : utilization(kind, tails, { demo, theme })
   }
 
   const gridForA = async (kind: KindA): Promise<null | GridSpec | GridSpec[]> => {
     if (kind === "all") {
-      const grids = await Promise.all([gridFor("status"), gridFor("cpu"), gridFor("memory")])
+      const grids = await Promise.all([
+        gridFor("status"),
+        gridFor("cpu%"),
+        gridFor("mem%"),
+        gridFor("gpu%"),
+        gridFor("gpumem%"),
+      ])
       return grids.filter(Boolean)
-    } else if (kind === "status" || kind === "cpu" || kind === "memory") {
+    } else if (isSupportedGrid(kind)) {
       return gridFor(kind)
     } else {
       return null

plugins/plugin-codeflare-dashboard/src/controller/dashboard/index.ts

@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 
-type Kind = "status" | "gpu" | "cpu" | "memory" | "logs"
+import type { SupportedGrid } from "./grids.js"
+
+type Kind = SupportedGrid | "logs"
 export type KindA = Kind | "all"
 export default Kind
 
@@ -26,9 +28,10 @@ export const resourcePaths: Record<Kind, string[]> = {
     "events/runtime-env-setup.txt",
     "logs/job.txt",
   ],
-  gpu: ["resources/gpu.txt"],
-  cpu: ["resources/pod-vmstat.txt"],
-  memory: ["resources/pod-memory.txt"],
+  "gpu%": ["resources/gpu.txt"],
+  "gpumem%": ["resources/gpu.txt"],
+  "cpu%": ["resources/pod-vmstat.txt"],
+  "mem%": ["resources/pod-memory.txt"],
   logs: ["logs/job.txt"],
 }
 

plugins/plugin-codeflare-dashboard/src/controller/dashboard/kinds.ts

@@ -21,19 +21,19 @@ import type { TextProps } from "ink"
 export type Theme = [TextProps, TextProps, TextProps, TextProps, TextProps]
 
 export const red: Theme = [
-  { color: "#b30000" },
-  { color: "#e34a33" },
-  { color: "#fc8d59" },
-  { color: "#fdbb84" },
   { color: "#fdd49e" },
+  { color: "#fdbb84" },
+  { color: "#fc8d59" },
+  { color: "#e34a33" },
+  { color: "#b30000" },
 ]
 
 export const purple: Theme = [
-  { color: "#810f7c" },
-  { color: "#8856a7" },
-  { color: "#8c96c6" },
-  { color: "#9ebcda" },
   { color: "#bfd3e6" },
+  { color: "#9ebcda" },
+  { color: "#8c96c6" },
+  { color: "#8856a7" },
+  { color: "#810f7c" },
 ]
 
 /*
@@ -47,35 +47,35 @@ export const magenta: Theme = [
 */
 
 export const magenta: Theme = [
-  { color: "#7a0177" },
-  { color: "#c51b8a" },
-  { color: "#f768a1" },
-  { color: "#fa9fb5" },
   { color: "#fcc5c0" },
+  { color: "#fa9fb5" },
+  { color: "#f768a1" },
+  { color: "#c51b8a" },
+  { color: "#7a0177" },
 ]
 
 export const green: Theme = [
-  { color: "#006837" },
-  { color: "#31a354" },
-  { color: "#78c679" },
-  { color: "#addd8e" },
   { color: "#d9f0a3" },
+  { color: "#addd8e" },
+  { color: "#78c679" },
+  { color: "#31a354" },
+  { color: "#006837" },
 ]
 
 export const blue: Theme = [
-  { color: "#045a8d" },
-  { color: "#2b8cbe" },
-  { color: "#74a9cf" },
-  { color: "#a6bddb" },
   { color: "#d0d1e6" },
+  { color: "#a6bddb" },
+  { color: "#74a9cf" },
+  { color: "#2b8cbe" },
+  { color: "#045a8d" },
 ]
 
 export const rain: Theme = [
-  { color: "#016c59" },
-  { color: "#1c9099" },
-  { color: "#67a9cf" },
-  { color: "#a6bddb" },
   { color: "#d0d1e6" },
+  { color: "#a6bddb" },
+  { color: "#67a9cf" },
+  { color: "#1c9099" },
+  { color: "#016c59" },
 ]
 
 export const themes = {

plugins/plugin-codeflare-dashboard/src/controller/dashboard/themes/utilization.ts

@@ -21,10 +21,11 @@ import type { Tail } from "./tailf.js"
 import type Options from "./options.js"
 import { isValidTheme, themes } from "./themes/utilization.js"
 import { OnData, Worker, GridSpec } from "../../components/Dashboard/index.js"
+import { SupportedUtilizationGrid, defaultUtilizationThemes, providerFor } from "./grids.js"
 
-type WorkerState = "0-20%" | "20-40%" | "40-60%" | "60-80%" | "80-100%"
+type WorkerState = "<20%" | "<40%" | "<60%" | "<80%" | "<100%"
 
-const states: WorkerState[] = ["0-20%", "20-40%", "40-60%", "60-80%", "80-100%"]
+const states: WorkerState[] = ["<20%", "<40%", "<60%", "<80%", "<100%"]
 
 /**
  * Maintain a model of live data from a given set of file streams
@@ -38,18 +39,27 @@ class Live {
   private stateFor(util: string): WorkerState {
     const percent = parseInt(util.replace(/%$/, ""), 10)
     const bucketWidth = ~~(100 / states.length)
-    return states[~~(percent / bucketWidth)]
+    return states[Math.min(~~(percent / bucketWidth), states.length - 1)]
   }
 
-  public constructor(private readonly tails: Promise<Tail>[], cb: OnData, styleOf: Record<WorkerState, TextProps>) {
+  public constructor(
+    expectedProvider: string,
+    private readonly tails: Promise<Tail>[],
+    cb: OnData,
+    styleOf: Record<WorkerState, TextProps>
+  ) {
     tails.map((tailf) => {
       tailf.then(({ stream }) => {
         stream.on("data", (data) => {
           if (data) {
             const line = stripAnsi(data)
             const cols = line.split(/\s+/)
 
-            const provider = !cols[0] ? undefined : cols[0].replace(/^\[/, "")
+            const provider = (!cols[0] ? undefined : cols[0].replace(/^\[/, "")) + (cols[1] ? " " + cols[1] : "")
+            if (provider !== expectedProvider) {
+              return
+            }
+
             const key = !cols[2] ? undefined : cols[2].replace(/\]$/, "")
             const metric = !provider || !key ? undefined : this.stateFor(key)
             const name = cols[3] ? cols[3].trim() : undefined
@@ -61,7 +71,7 @@ class Live {
             } else if (!metric) {
               // ignoring this line
               return
-            } else if (provider === "Workers" && (!/^pod\//.test(name) || /cleaner/.test(name))) {
+            } else if (!/^pod\//.test(name) || /cleaner/.test(name)) {
               // only track pod events, and ignore our custodial pods
               return
             } else {
@@ -178,11 +188,11 @@ function capitalize(str: string) {
 }
 
 export default function utilizationDashboard(
-  kind: "cpu" | "memory",
+  kind: SupportedUtilizationGrid,
   tails: Promise<Tail>[],
-  opts: Pick<Options, "demo" | "theme" | "themeDefault">
+  opts: Pick<Options, "demo" | "theme"> & Partial<Pick<Options, "themeDefault">>
 ): GridSpec {
-  const { theme: themeS = opts.themeDefault } = opts
+  const { theme: themeS = opts.themeDefault || defaultUtilizationThemes[kind] } = opts
   if (!isValidTheme(themeS)) {
     throw new Error("Invalid theme: " + themeS)
   }
@@ -198,7 +208,8 @@ export default function utilizationDashboard(
     if (opts.demo) {
       return new Demo(cb, styleOf)
     } else {
-      return new Live(tails, cb, styleOf)
+      const expectedProvider = providerFor[kind]
+      return new Live(expectedProvider, tails, cb, styleOf)
     }
   }
 

plugins/plugin-codeflare-dashboard/src/controller/dashboard/utilization.tsx

@@ -51,7 +51,7 @@ export default async function dump(args: Arguments<Options>) {
   if (kind === "path") {
     // print the path to the data captured for the given jobId in the given profile
     const { dirname } = await import("path")
-    return Array.from(new Set(await pathsFor("cpu", profile, jobId).then((_) => _.map((_) => dirname(_)))))[0]
+    return Array.from(new Set(await pathsFor("cpu%", profile, jobId).then((_) => _.map((_) => dirname(dirname(_))))))[0]
   } else if (!args.parsedOptions.f) {
     const { createReadStream } = await import("fs")
     await Promise.all(

plugins/plugin-codeflare-dashboard/src/controller/dump.ts

@@ -30,7 +30,7 @@
     "@types/split2": "^3.2.1"
   },
   "dependencies": {
-    "@guidebooks/store": "^7.4.3",
+    "@guidebooks/store": "^7.4.4",
     "@logdna/tail-file": "^3.0.1",
     "@patternfly/react-charts": "^6.94.18",
     "@patternfly/react-core": "^4.276.6",