Skip to content

Commit 00000ad

Browse files
rzabarazeshrzabarazesh
committed
Add per job runtime metrics and docker build metrics
1 parent 0f66519 commit 00000ad

File tree

9 files changed

+707
-37
lines changed

9 files changed

+707
-37
lines changed

‎torchci/clickhouse_queries/vllm/docker_build_runtime/params.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"params": {
3+
"repo": "String",
4+
"startTime": "DateTime64(3)",
5+
"stopTime": "DateTime64(3)"
6+
},
7+
"tests": [
8+
{
9+
"repo": "https://github.com/vllm-project/vllm.git",
10+
"startTime": "2025-10-01T00:00:00.000",
11+
"stopTime": "2025-11-01T00:00:00.000"
12+
}
13+
]
14+
}
15+

‎torchci/clickhouse_queries/vllm/docker_build_runtime/query.sql‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
-- vLLM Docker Build Image Runtime Trends (main branch only)
2+
-- Tracks runtime for the ":docker: build image" job specifically
3+
-- This is a critical job for build speed monitoring
4+
5+
WITH jobs AS (
6+
SELECT
7+
tupleElement(job, 'name') AS job_name,
8+
tupleElement(job, 'started_at') AS job_started_at,
9+
tupleElement(job, 'finished_at') AS job_finished_at,
10+
tupleElement(job, 'state') AS job_state,
11+
tupleElement(build, 'number') AS build_number
12+
FROM vllm.vllm_buildkite_jobs
13+
WHERE
14+
tupleElement(pipeline, 'repository') = {repo: String }
15+
AND tupleElement(build, 'branch') = 'main'
16+
AND tupleElement(job, 'name') = ':docker: build image'
17+
AND tupleElement(job, 'started_at') IS NOT NULL
18+
AND tupleElement(job, 'finished_at') IS NOT NULL
19+
AND tupleElement(job, 'started_at') >= {startTime: DateTime64(3) }
20+
AND tupleElement(job, 'started_at') < {stopTime: DateTime64(3) }
21+
AND lowerUTF8(tupleElement(job, 'state')) IN ('passed', 'finished', 'success', 'failed')
22+
)
23+
24+
SELECT
25+
job_started_at AS timestamp,
26+
build_number,
27+
round(dateDiff('second', job_started_at, job_finished_at) / 60.0, 2) AS runtime_minutes
28+
FROM jobs
29+
ORDER BY job_started_at ASC
30+

‎torchci/clickhouse_queries/vllm/job_runtime_trends/params.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"params": {
3+
"repo": "String",
4+
"startTime": "DateTime64(3)",
5+
"stopTime": "DateTime64(3)",
6+
"jobGroups": "Array(String)"
7+
},
8+
"tests": [
9+
{
10+
"repo": "https://github.com/vllm-project/vllm.git",
11+
"startTime": "2025-10-01T00:00:00.000",
12+
"stopTime": "2025-10-08T00:00:00.000",
13+
"jobGroups": ["main", "amd", "torch_nightly"]
14+
}
15+
]
16+
}
17+

‎torchci/clickhouse_queries/vllm/job_runtime_trends/query.sql‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
-- vLLM Job Runtime Trends (main branch only)
2+
-- Aggregates per-job runtime statistics by day
3+
-- Shows count, mean, p90, and max runtime for each job per day
4+
-- Supports filtering by job groups: AMD, Torch Nightly, or Main
5+
6+
WITH jobs AS (
7+
SELECT
8+
tupleElement(job, 'name') AS job_name,
9+
tupleElement(job, 'started_at') AS job_started_at,
10+
tupleElement(job, 'finished_at') AS job_finished_at,
11+
tupleElement(job, 'state') AS job_state,
12+
tupleElement(build, 'branch') AS branch
13+
FROM vllm.vllm_buildkite_jobs
14+
WHERE
15+
tupleElement(pipeline, 'repository') = {repo: String }
16+
AND tupleElement(build, 'branch') = 'main'
17+
AND tupleElement(job, 'started_at') IS NOT NULL
18+
AND tupleElement(job, 'finished_at') IS NOT NULL
19+
AND tupleElement(job, 'started_at') >= {startTime: DateTime64(3) }
20+
AND tupleElement(job, 'started_at') < {stopTime: DateTime64(3) }
21+
AND lowerUTF8(tupleElement(job, 'state')) IN ('passed', 'finished', 'success', 'failed')
22+
-- Job group filtering: AMD, Torch Nightly, or Main
23+
AND (
24+
(
25+
has({jobGroups: Array(String)}, 'amd')
26+
AND positionCaseInsensitive(tupleElement(job, 'name'), 'AMD')
27+
> 0
28+
)
29+
OR (
30+
has({jobGroups: Array(String)}, 'torch_nightly')
31+
AND positionCaseInsensitive(
32+
tupleElement(job, 'name'), 'Torch Nightly'
33+
)
34+
> 0
35+
)
36+
OR (
37+
has({jobGroups: Array(String)}, 'main')
38+
AND positionCaseInsensitive(tupleElement(job, 'name'), 'AMD')
39+
= 0
40+
AND positionCaseInsensitive(
41+
tupleElement(job, 'name'), 'Torch Nightly'
42+
)
43+
= 0
44+
)
45+
)
46+
)
47+
48+
SELECT
49+
job_name,
50+
toDate(job_started_at) AS date,
51+
count() AS count,
52+
round(avg(dateDiff('second', job_started_at, job_finished_at) / 60.0), 2) AS mean_runtime_minutes,
53+
round(quantile(0.9)(dateDiff('second', job_started_at, job_finished_at) / 60.0), 2) AS p90_runtime_minutes,
54+
round(max(dateDiff('second', job_started_at, job_finished_at) / 60.0), 2) AS max_runtime_minutes
55+
FROM jobs
56+
GROUP BY job_name, date
57+
ORDER BY job_name ASC, date ASC
58+

‎torchci/components/metrics/vllm/CiDurationsPanel.tsx‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -230,28 +230,11 @@ export default function CiDurationsPanel({
230230
...getLineSeries(dailyMeanSuccess, dailyMeanNonCanceled),
231231
...getScatterSeriesByState(source),
232232
],
233-
dataZoom: [
234-
{
235-
type: "slider",
236-
show: true,
237-
xAxisIndex: 0,
238-
bottom: 0,
239-
start: 0,
240-
end: 100,
241-
height: 25,
242-
},
243-
{
244-
type: "inside",
245-
xAxisIndex: 0,
246-
start: 0,
247-
end: 100,
248-
},
249-
],
250233
};
251234

252235
return (
253236
<ChartPaper
254-
tooltip="Main branch CI runtimes over time. Green line = mean runtime for successful builds, Pink line = mean including failures. Scatter points = individual builds (click to view in Buildkite). Use slider or scroll to zoom."
237+
tooltip="Main branch CI runtimes over time. Green line = mean runtime for successful builds, Pink line = mean including failures. Scatter points = individual builds (click to view in Buildkite)."
255238
option={options}
256239
onEvents={{
257240
click: handleBuildClick,

‎torchci/components/metrics/vllm/DockerBuildRuntimePanel.tsx‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import dayjs from "dayjs";
2+
import { EChartsOption } from "echarts";
3+
import { useDarkMode } from "lib/DarkModeContext";
4+
import _ from "lodash";
5+
import { ChartPaper } from "./chartUtils";
6+
import { COLOR_SUCCESS, COLOR_WARNING } from "./constants";
7+
8+
interface DockerBuildData {
9+
timestamp: string;
10+
build_number: number;
11+
runtime_minutes: number;
12+
}
13+
14+
// Helper function to format tooltip
15+
function formatTooltip(params: any): string {
16+
if (!params || !params.data) return "";
17+
18+
const data = params.data;
19+
20+
// Handle both scatter (array) and line (object) series
21+
let timestamp, runtime, buildNumber;
22+
23+
if (Array.isArray(data)) {
24+
timestamp = data[0];
25+
runtime = data[1];
26+
buildNumber = data[2];
27+
} else {
28+
// For line series (daily average)
29+
timestamp = data.day;
30+
runtime = data.value;
31+
buildNumber = null;
32+
}
33+
34+
if (!timestamp || runtime === undefined) return "";
35+
36+
const formattedTime = dayjs(timestamp).format("M/D/YY h:mm A");
37+
38+
let result = buildNumber
39+
? `<b>Build #${buildNumber}</b><br/>`
40+
: `<b>Daily Average</b><br/>`;
41+
result += `Time: ${formattedTime}<br/>`;
42+
result += `Runtime: <b>${runtime.toFixed(1)} min</b>`;
43+
44+
return result;
45+
}
46+
47+
// Helper function to handle click events
48+
function handleBuildClick(params: any) {
49+
if (params?.componentType === "series") {
50+
const data = Array.isArray(params.data) ? params.data : [params.data];
51+
const buildNumber = data[2];
52+
if (buildNumber !== undefined && buildNumber !== null) {
53+
const url = `https://buildkite.com/vllm/ci/builds/${buildNumber}/`;
54+
if (typeof window !== "undefined") {
55+
window.open(url, "_blank");
56+
}
57+
}
58+
}
59+
}
60+
61+
export default function DockerBuildRuntimePanel({
62+
data,
63+
}: {
64+
data: DockerBuildData[] | undefined;
65+
}) {
66+
const { darkMode } = useDarkMode();
67+
68+
// Process data for chart
69+
const chartData = (data || []).map((d) => [
70+
dayjs(d.timestamp).toISOString(),
71+
d.runtime_minutes,
72+
d.build_number,
73+
]);
74+
75+
// Calculate daily average for trend line
76+
const groupedByDay = _.groupBy(data || [], (d) =>
77+
dayjs(d.timestamp).format("YYYY-MM-DD")
78+
);
79+
80+
const dailyAvg = Object.entries(groupedByDay)
81+
.map(([day, records]) => {
82+
const avgRuntime = _.meanBy(records, "runtime_minutes");
83+
return {
84+
day,
85+
value: Number(avgRuntime.toFixed(1)),
86+
};
87+
})
88+
.sort((a, b) => (a.day < b.day ? -1 : 1));
89+
90+
// Calculate statistics
91+
const runtimes = (data || []).map((d) => d.runtime_minutes);
92+
const avgRuntime = runtimes.length ? _.mean(runtimes).toFixed(1) : "N/A";
93+
const p90Runtime = runtimes.length
94+
? runtimes.sort((a, b) => a - b)[
95+
Math.floor(runtimes.length * 0.9)
96+
].toFixed(1)
97+
: "N/A";
98+
99+
const options: EChartsOption = {
100+
title: {
101+
text: "Docker Build Image Runtime",
102+
subtext: `Avg: ${avgRuntime}m | P90: ${p90Runtime}m | Total builds: ${runtimes.length}`,
103+
textStyle: {
104+
fontSize: 14,
105+
},
106+
},
107+
legend: {
108+
top: 24,
109+
data: ["Individual Builds", "Daily Average"],
110+
},
111+
grid: { top: 60, right: 20, bottom: 80, left: 60 },
112+
dataset: [{ source: chartData }, { source: dailyAvg }],
113+
xAxis: {
114+
type: "time",
115+
axisLabel: {
116+
hideOverlap: true,
117+
formatter: (value: number) => dayjs(value).format("M/D"),
118+
},
119+
},
120+
yAxis: {
121+
type: "value",
122+
name: "Runtime (minutes)",
123+
nameLocation: "middle",
124+
nameGap: 45,
125+
nameRotate: 90,
126+
axisLabel: {
127+
formatter: (value: number) => `${value}m`,
128+
},
129+
},
130+
series: [
131+
{
132+
name: "Individual Builds",
133+
type: "scatter",
134+
datasetIndex: 0,
135+
symbolSize: 6,
136+
itemStyle: { color: COLOR_SUCCESS, opacity: 0.6 },
137+
},
138+
{
139+
name: "Daily Average",
140+
type: "line",
141+
datasetIndex: 1,
142+
smooth: true,
143+
encode: { x: "day", y: "value" },
144+
lineStyle: { color: COLOR_WARNING, width: 2 },
145+
itemStyle: { color: COLOR_WARNING },
146+
showSymbol: true,
147+
symbolSize: 4,
148+
},
149+
],
150+
tooltip: {
151+
trigger: "item",
152+
formatter: formatTooltip,
153+
},
154+
};
155+
156+
return (
157+
<ChartPaper
158+
tooltip="Docker build image runtime over time. Each point is an individual build (click to open in Buildkite). Green line shows daily average trend."
159+
option={options}
160+
onEvents={{ click: handleBuildClick }}
161+
darkMode={darkMode}
162+
/>
163+
);
164+
}
165+

‎torchci/components/metrics/vllm/DurationDistributionPanel.tsx‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -164,28 +164,11 @@ export default function DurationDistributionPanel({
164164
axisPointer: { type: "shadow" },
165165
formatter: formatDistributionTooltip,
166166
},
167-
dataZoom: [
168-
{
169-
type: "slider",
170-
show: true,
171-
xAxisIndex: 0,
172-
bottom: 0,
173-
start: 0,
174-
end: 100,
175-
height: 25,
176-
},
177-
{
178-
type: "inside",
179-
xAxisIndex: 0,
180-
start: 0,
181-
end: 100,
182-
},
183-
],
184167
};
185168

186169
return (
187170
<ChartPaper
188-
tooltip="Histogram showing distribution of main branch CI runtimes (how long builds take to complete). Green = successful builds, Red = failed builds, Gray = canceled builds. Use slider or scroll to zoom."
171+
tooltip="Histogram showing distribution of main branch CI runtimes (how long builds take to complete). Green = successful builds, Red = failed builds, Gray = canceled builds."
189172
option={options}
190173
darkMode={darkMode}
191174
/>

0 commit comments

Comments
 (0)