Skip to content

Commit afdad22

Browse files
craig[bot]dt
craig[bot]
and
dt
committed
152441: scripts: add job progress plot utility r=dt a=dt Generated by Claude. Release Notes: None Epic: None 152442: scripts: add roachtest artifact fetcher r=dt a=dt Drops them in artifacts/roachtest-[testname]-[issue] Usable on its own, but also usable by claude. Generated by Claude. Release note: none Epic: none Co-authored-by: David Taylor <tinystatemachine@gmail.com>
3 parents a2a8c32 + 64f31de + 02675a5 commit afdad22

File tree

2 files changed

+413
-0
lines changed

2 files changed

+413
-0
lines changed

‎scripts/fetch-roachtest-artifacts‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
#!/usr/bin/env bash
2+
#
3+
# fetch-roachtest-artifacts - Fetch artifacts for a roachtest issue
4+
#
5+
set -euo pipefail
6+
7+
usage() {
8+
echo "Usage: $0 <issue-number-or-url-or-teamcity-url>"
9+
echo ""
10+
echo "Examples:"
11+
echo " $0 12345"
12+
echo " $0 https://github.com/cockroachdb/cockroach/issues/12345"
13+
echo " $0 https://github.com/cockroachdb/cockroach/issues/12345#issuecomment-67890"
14+
echo " $0 'https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_Nightlies_RoachtestWeeklyBazel/20321836?buildTab=artifacts#/c2c/import/7tb/kv0'"
15+
exit 1
16+
}
17+
18+
# URL encode special characters for TeamCity API (but not slashes)
19+
url_encode() {
20+
local string="$1"
21+
echo "$string" | sed 's/=/%3D/g; s/ /%20/g'
22+
}
23+
24+
# Recursively download artifacts from TeamCity
25+
download_artifacts_recursive() {
26+
local build_id="$1"
27+
local path="$2"
28+
local dest_dir="$3"
29+
local depth="${4:-0}"
30+
31+
# Prevent infinite recursion
32+
[[ $depth -gt 10 ]] && return
33+
34+
# Get children of current path
35+
local encoded_path
36+
encoded_path=$(url_encode "$path")
37+
local children_xml
38+
children_xml=$(curl -s "https://teamcity.cockroachdb.com/guestAuth/app/rest/builds/id:$build_id/artifacts/children/$encoded_path/" 2>/dev/null)
39+
40+
# Debug output
41+
if [[ -n "${ROACHTEST_DEBUG:-}" ]]; then
42+
echo "DEBUG: Checking path: $path (encoded: $encoded_path)"
43+
echo "DEBUG: XML response: ${children_xml:0:200}..."
44+
fi
45+
46+
# Check if we got valid XML
47+
if [[ -z "$children_xml" || "$children_xml" == *"<error"* || "$children_xml" == *"HTTP 400"* || "$children_xml" == *"status code: 400"* || "$children_xml" == *"404"* || "$children_xml" == *"Build not found"* ]]; then
48+
# If this is the root path (depth 0), it's a fatal error
49+
if [[ $depth -eq 0 ]]; then
50+
echo "Error: Build $build_id not found or path $path does not exist"
51+
exit 1
52+
fi
53+
return
54+
fi
55+
56+
# Extract all items (files and directories)
57+
local items
58+
items=$(echo "$children_xml" | grep -o 'name="[^"]*"' | cut -d'"' -f2)
59+
60+
if [[ -n "${ROACHTEST_DEBUG:-}" ]]; then
61+
echo "DEBUG: Found items: $items"
62+
fi
63+
64+
[[ -z "$items" ]] && return
65+
66+
local pids=()
67+
while read -r item; do
68+
[[ -z "$item" ]] && continue
69+
70+
local item_path="$path/$item"
71+
local encoded_item_path
72+
encoded_item_path=$(url_encode "$item_path")
73+
74+
# Check if this item has children (is a directory) first
75+
local item_children
76+
item_children=$(curl -s "https://teamcity.cockroachdb.com/guestAuth/app/rest/builds/id:$build_id/artifacts/children/$encoded_item_path/" 2>/dev/null)
77+
78+
if [[ -n "${ROACHTEST_DEBUG:-}" ]]; then
79+
echo "DEBUG: Checking if $item is directory. Response: ${item_children:0:100}..."
80+
fi
81+
82+
if [[ -n "$item_children" && "$item_children" != *"<error"* && "$item_children" != *"HTTP 400"* && "$item_children" != *"status code: 400"* && "$item_children" != *"status code: 404"* ]] && [[ "$item" != *.zip ]]; then
83+
# It's a directory (not a zip file), recurse into it
84+
if [[ -n "${ROACHTEST_DEBUG:-}" ]]; then
85+
echo "DEBUG: Recursing into directory: $item"
86+
fi
87+
download_artifacts_recursive "$build_id" "$item_path" "$dest_dir" $((depth + 1))
88+
elif [[ "$item" == *.* ]]; then
89+
# It's a file, download it
90+
(
91+
# Create subdirectory structure relative to the original path
92+
local relative_path="${item_path#$2/}" # Remove the root path prefix
93+
local file_dir="$dest_dir/$(dirname "$relative_path")"
94+
mkdir -p "$file_dir"
95+
local file_path="$dest_dir/$relative_path"
96+
97+
local file_url="https://teamcity.cockroachdb.com/guestAuth/app/rest/builds/id:$build_id/artifacts/content/$encoded_item_path"
98+
if [[ -n "${ROACHTEST_DEBUG:-}" ]]; then
99+
echo "DEBUG: Downloading $file_url to $file_path"
100+
fi
101+
if curl -f -s "$file_url" -o "$file_path" 2>/dev/null && [[ -s "$file_path" ]]; then
102+
echo "Downloaded: $relative_path"
103+
104+
# Only unzip actual zip files
105+
if [[ "$item" == *.zip ]] && file "$file_path" | grep -q "Zip archive"; then
106+
local extract_dir="${file_path%.zip}"
107+
mkdir -p "$extract_dir"
108+
(cd "$extract_dir" && unzip -o -q "../$(basename "$file_path")")
109+
110+
# Flatten if archive contains single directory
111+
local contents=($(ls "$extract_dir"))
112+
if [[ ${#contents[@]} -eq 1 && -d "$extract_dir/${contents[0]}" ]]; then
113+
mv "$extract_dir/${contents[0]}"/* "$extract_dir/" 2>/dev/null || true
114+
rmdir "$extract_dir/${contents[0]}" 2>/dev/null || true
115+
fi
116+
117+
rm "$file_path"
118+
fi
119+
fi
120+
) &
121+
pids+=($!)
122+
else
123+
# It's a file without an extension, download it
124+
(
125+
# Create subdirectory structure relative to the original path
126+
local relative_path="${item_path#$2/}" # Remove the root path prefix
127+
local file_dir="$dest_dir/$(dirname "$relative_path")"
128+
mkdir -p "$file_dir"
129+
local file_path="$dest_dir/$relative_path"
130+
131+
local file_url="https://teamcity.cockroachdb.com/guestAuth/app/rest/builds/id:$build_id/artifacts/content/$encoded_item_path"
132+
if [[ -n "${ROACHTEST_DEBUG:-}" ]]; then
133+
echo "DEBUG: Downloading $file_url to $file_path"
134+
fi
135+
if curl -f -s "$file_url" -o "$file_path" 2>/dev/null && [[ -s "$file_path" ]]; then
136+
echo "Downloaded: $relative_path"
137+
fi
138+
) &
139+
pids+=($!)
140+
fi
141+
done <<< "$items"
142+
143+
# Wait for all downloads to complete
144+
if [[ ${#pids[@]} -gt 0 ]]; then
145+
for pid in "${pids[@]}"; do
146+
wait "$pid"
147+
done
148+
fi
149+
}
150+
151+
# Handle direct TeamCity artifact URLs
152+
handle_teamcity_url() {
153+
local tc_url="$1"
154+
155+
# Extract build ID and path from TeamCity URL
156+
local build_id path
157+
build_id=$(echo "$tc_url" | grep -o '[0-9]\+' | head -1)
158+
159+
if [[ "$tc_url" == *"#/"* ]]; then
160+
path=$(echo "$tc_url" | sed 's/.*#\///')
161+
else
162+
echo "Error: TeamCity URL must include a path after #/"
163+
echo "Expected format: https://teamcity.cockroachdb.com/.../BUILD_ID?buildTab=artifacts#/path"
164+
echo "Got: $tc_url"
165+
exit 1
166+
fi
167+
168+
if [[ -z "$build_id" || -z "$path" ]]; then
169+
echo "Error: Could not extract build ID or path from TeamCity URL: $tc_url"
170+
echo "Expected format: https://teamcity.cockroachdb.com/.../BUILD_ID?buildTab=artifacts#/path"
171+
exit 1
172+
fi
173+
174+
# Extract test name from path for directory naming
175+
local test_name
176+
test_name=$(echo "$path" | tr '/' '-' | tr -cd 'a-zA-Z0-9_-')
177+
test_name="${test_name:-unknown}"
178+
179+
# Create artifacts directory
180+
local artifacts_dir="artifacts/teamcity-${test_name}-${build_id}"
181+
mkdir -p "$artifacts_dir"
182+
echo "Fetching TeamCity artifacts from build $build_id path $path"
183+
echo "Downloading to $artifacts_dir"
184+
185+
# Download artifacts
186+
download_artifacts_recursive "$build_id" "$path" "$artifacts_dir"
187+
188+
# Verify at least artifacts.zip was downloaded (could be nested in subdirectories)
189+
if ! find "$artifacts_dir" -name "artifacts.zip" -o -name "artifacts" -type d | grep -q .; then
190+
echo "Error: Failed to download artifacts.zip or extract artifacts directory"
191+
echo "Build $build_id may not have artifacts at path $path, or artifacts may have expired"
192+
exit 1
193+
fi
194+
195+
echo "TeamCity artifacts successfully downloaded to $artifacts_dir"
196+
exit 0
197+
}
198+
199+
main() {
200+
[[ $# -eq 0 || "$1" == "--help" ]] && usage
201+
202+
# Check if this is a direct TeamCity URL
203+
if [[ "$1" == *"teamcity.cockroachdb.com"* ]]; then
204+
if [[ "$1" == *"buildTab=artifacts"* ]]; then
205+
handle_teamcity_url "$1"
206+
return
207+
else
208+
echo "Error: TeamCity URL must include 'buildTab=artifacts'"
209+
echo "Expected format: https://teamcity.cockroachdb.com/.../BUILD_ID?buildTab=artifacts#/path"
210+
exit 1
211+
fi
212+
fi
213+
214+
# Extract issue number and comment ID (if present)
215+
local issue_number comment_id=""
216+
217+
if [[ "$1" == *"/issues/"* ]]; then
218+
# GitHub URL: extract path after github.com/cockroachdb/cockroach/
219+
local path="${1#*github.com/cockroachdb/cockroach/}"
220+
if [[ "$path" == issues/* ]]; then
221+
# Remove issues/ prefix and split on #
222+
local remainder="${path#issues/}"
223+
issue_number="${remainder%%#*}"
224+
225+
# Check for comment ID
226+
if [[ "$remainder" == *"#issuecomment-"* ]]; then
227+
comment_id="${remainder#*#issuecomment-}"
228+
fi
229+
fi
230+
else
231+
# Just a number: 12345
232+
issue_number="${1##*/}"
233+
issue_number="${issue_number%)*}"
234+
issue_number="${issue_number%#*}"
235+
fi
236+
237+
[[ ! "$issue_number" =~ ^[0-9]+$ ]] && { echo "Error: Invalid issue number from '$1'"; exit 1; }
238+
239+
# Get issue data from GitHub API
240+
local issue_data
241+
issue_data=$(curl -s "https://api.github.com/repos/cockroachdb/cockroach/issues/$issue_number") || {
242+
echo "Error: Failed to fetch issue #$issue_number"
243+
exit 1
244+
}
245+
246+
# Check if issue exists
247+
if echo "$issue_data" | jq -e '.message // empty' >/dev/null 2>&1; then
248+
echo "Error: $(echo "$issue_data" | jq -r '.message')"
249+
exit 1
250+
fi
251+
252+
# Extract info using jq and basic string ops
253+
local title body url test_name
254+
title=$(echo "$issue_data" | jq -r '.title')
255+
url=$(echo "$issue_data" | jq -r '.html_url')
256+
257+
# Get body from specific comment or issue
258+
if [[ -n "$comment_id" ]]; then
259+
echo "Fetching comment #$comment_id from issue #$issue_number"
260+
local comment_data
261+
comment_data=$(curl -s "https://api.github.com/repos/cockroachdb/cockroach/issues/comments/$comment_id") || {
262+
echo "Error: Failed to fetch comment #$comment_id"
263+
exit 1
264+
}
265+
266+
if echo "$comment_data" | jq -e '.message // empty' >/dev/null 2>&1; then
267+
echo "Error: $(echo "$comment_data" | jq -r '.message')"
268+
exit 1
269+
fi
270+
271+
body=$(echo "$comment_data" | jq -r '.body')
272+
url="${url}#issuecomment-${comment_id}"
273+
else
274+
echo "Fetching issue #$issue_number"
275+
body=$(echo "$issue_data" | jq -r '.body')
276+
fi
277+
test_name=$(echo "$title" | sed -n 's/.*roachtest[: ]*\([^ ]*\) .*/\1/p' | head -1)
278+
test_name="${test_name:-unknown}"
279+
test_name=$(echo "$test_name" | tr '/' '-' | tr -cd 'a-zA-Z0-9_-=')
280+
281+
# Create artifacts directory
282+
local artifacts_dir="artifacts/roachtest-${test_name}-${issue_number}"
283+
if [[ -n "$comment_id" ]]; then
284+
artifacts_dir="artifacts/roachtest-${test_name}-${issue_number}-comment-${comment_id}"
285+
fi
286+
mkdir -p "$artifacts_dir"
287+
echo "fetching to $artifacts_dir"
288+
289+
# Find and download TeamCity artifacts
290+
local downloaded_files=()
291+
while read -r tc_url; do
292+
[[ -z "$tc_url" || ! "$tc_url" =~ buildTab=artifacts ]] && continue
293+
294+
# Transform: buildConfiguration/.../ID?buildTab=artifacts#/path → API download
295+
local build_id path
296+
build_id=$(echo "$tc_url" | grep -o '[0-9]\+' | head -1)
297+
path=$(echo "$tc_url" | sed 's/.*#\///')
298+
299+
if [[ -n "$build_id" && -n "$path" ]]; then
300+
download_artifacts_recursive "$build_id" "$path" "$artifacts_dir"
301+
downloaded_files+=("$artifacts_dir")
302+
fi
303+
done <<< "$(echo "$body" | grep -o 'https://teamcity[^[:space:]]*' | sed 's/)$//' || true)"
304+
305+
# Output summary and verification
306+
# Always verify artifacts were actually downloaded, regardless of downloaded_files array
307+
if ! find "$artifacts_dir" -name "artifacts.zip" -o -name "artifacts" -type d | grep -q .; then
308+
if [ ${#downloaded_files[@]} -gt 0 ]; then
309+
echo "Error: #$issue_number - $title: failed to download artifacts.zip"
310+
echo "TeamCity artifacts may have expired or the build may not contain test artifacts"
311+
else
312+
echo "Error: #$issue_number - $title: no artifacts found"
313+
echo "This could happen if:"
314+
echo " - TeamCity artifacts have expired"
315+
echo " - Issue has no TeamCity links"
316+
echo " - URLs don't match expected format"
317+
fi
318+
exit 1
319+
fi
320+
echo "#$issue_number - $title: all artifacts downloaded to $artifacts_dir"
321+
}
322+
323+
main "$@"

0 commit comments

Comments
 (0)