Skip to content

Commit 31b2ad9

Browse files
jsturtevantjsturtevant
committed
Refactor CI failure and add to nightly job
Signed-off-by: James Sturtevant <jsturtevant@gmail.com>
1 parent 511f34e commit 31b2ad9

File tree

4 files changed

+184
-155
lines changed

4 files changed

+184
-155
lines changed

‎.github/workflows/Fuzzing.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,6 @@ jobs:
2828
uses: actions/checkout@v5
2929

3030
- name: Notify Fuzzing Failure
31-
run: ./dev/notify-fuzzing-failure.sh "fuzz_host_print,fuzz_guest_call,fuzz_host_call"
31+
run: ./dev/notify-ci-failure.sh --labels="area/fuzzing,kind/bug,area/testing,lifecycle/needs-review"
3232
env:
3333
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/RustNightly.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ name: Nightly
55
on:
66
workflow_dispatch:
77
schedule:
8+
# 12:00 AM, every 2 days
89
- cron: '0 0 */2 * *'
910

1011
jobs:
@@ -23,5 +24,17 @@ jobs:
2324
config: ${{ matrix.config }}
2425
target_triple: x86_64-unknown-linux-musl
2526

26-
# TODO: using notification script
27+
notify-failure:
28+
runs-on: ubuntu-latest
29+
needs: musl
30+
if: always() && needs.musl.result == 'failure'
31+
permissions:
32+
issues: write
33+
steps:
34+
- name: Checkout code
35+
uses: actions/checkout@v5
2736

37+
- name: Notify Nightly Failure
38+
run: ./dev/notify-ci-failure.sh --title="Nightly musl Failure - ${{ github.run_number }}" --labels="area/ci-periodics,area/testing,kind/bug,lifecycle/needs-review"
39+
env:
40+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎dev/notify-ci-failure.sh‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#!/bin/bash
2+
set -e
3+
set -u
4+
set -o pipefail
5+
6+
## DESCRIPTION:
7+
##
8+
# Generic notifier for CI job failures that can create or update GitHub issues.
9+
## This script creates or updates GitHub issues when a jobs fail.
10+
## It checks for existing open failure issues and either creates
11+
## a new one or adds a comment to an existing one.
12+
##
13+
## PRE-REQS:
14+
##
15+
## This script assumes that the gh cli is installed and in the PATH
16+
## and that there is a GitHub PAT in the GITHUB_TOKEN env var
17+
## with the following permissions:
18+
## - issues (read/write)
19+
## or that the user is logged into the gh cli with an account with those permissions
20+
##
21+
## Usage examples:
22+
## ./dev/notify-fuzzing-failure.sh
23+
## ./dev/notify-fuzzing-failure.sh --title="Nightly Failure" --labels="area/testing,kind/bug"
24+
## ./dev/notify-fuzzing-failure.sh --test
25+
## Run this script locally like:
26+
## GITHUB_REPOSITORY="fork/hyperlight" GITHUB_RUN_ID=1 ./dev/notify-fuzzing-failure.sh --title="Nightly Failure" --labels="area/testing,kind/bug"
27+
28+
REPO="${GITHUB_REPOSITORY:-hyperlight-dev/hyperlight}"
29+
WORKFLOW_RUN_URL="${GITHUB_SERVER_URL:-https://github.com}/${REPO}/actions/runs/${GITHUB_RUN_ID:-unknown}"
30+
TEST_MODE=false
31+
ISSUE_TITLE=""
32+
LABELS="area/testing,kind/bug,area/fuzzing,lifecycle/needs-review"
33+
34+
for arg in "$@"; do
35+
case $arg in
36+
--test)
37+
TEST_MODE=true
38+
shift
39+
;;
40+
--title=*)
41+
ISSUE_TITLE="${arg#*=}"
42+
shift
43+
;;
44+
--labels=*)
45+
LABELS="${arg#*=}"
46+
shift
47+
;;
48+
*)
49+
esac
50+
done
51+
52+
# Normalize labels into an array
53+
IFS=',' read -r -a LABEL_ARRAY <<< "$LABELS"
54+
55+
# Choose a label to search existing issues for; prefer the first label if present
56+
SEARCH_LABEL="${LABEL_ARRAY[0]:-area/fuzzing}"
57+
58+
# Build issue title if not provided
59+
if [ -z "$ISSUE_TITLE" ]; then
60+
ISSUE_TITLE="Job Failure - $(date '+%Y-%m-%d')"
61+
fi
62+
63+
64+
if [ "$TEST_MODE" = true ]; then
65+
echo "✅ Running in test mode - script structure is valid"
66+
echo "Would check for issues in $REPO"
67+
echo "Workflow URL would be: $WORKFLOW_RUN_URL"
68+
echo "Issue Title would be: $ISSUE_TITLE"
69+
echo "Labels would be: $LABELS"
70+
echo "Search Label would be: $SEARCH_LABEL"
71+
exit 0
72+
fi
73+
74+
# Extract owner and repo name from the repository
75+
OWNER=$(echo "$REPO" | cut -d'/' -f1)
76+
REPO_NAME=$(echo "$REPO" | cut -d'/' -f2)
77+
78+
echo "Checking for existing issues in $REPO with label '$SEARCH_LABEL'..."
79+
EXISTING_ISSUES=$(gh api graphql -f query='
80+
query($owner: String!, $repo: String!, $label: String!) {
81+
repository(owner: $owner, name: $repo) {
82+
issues(first: 10, states: OPEN, labels: [$label]) {
83+
totalCount
84+
nodes {
85+
number
86+
title
87+
url
88+
labels(first: 20) {
89+
nodes {
90+
name
91+
}
92+
}
93+
}
94+
}
95+
}
96+
}' -f owner="$OWNER" -f repo="$REPO_NAME" -f label="$SEARCH_LABEL" --jq '.data.repository.issues') || EXISTING_ISSUES=""
97+
98+
FUZZING_ISSUES=$(echo "$EXISTING_ISSUES" | jq '.nodes[]' 2>/dev/null || echo "")
99+
FUZZING_ISSUE_COUNT=0
100+
if [ -n "$FUZZING_ISSUES" ]; then
101+
FUZZING_ISSUE_COUNT=$(echo "$FUZZING_ISSUES" | jq -s 'length' 2>/dev/null || echo "0")
102+
fi
103+
104+
echo "Found $FUZZING_ISSUE_COUNT existing issue(s) matching label '$SEARCH_LABEL'"
105+
106+
if [ "$FUZZING_ISSUE_COUNT" -gt 0 ]; then
107+
ISSUE_NUMBER=$(echo "$FUZZING_ISSUES" | jq -r '.number' | head -1)
108+
ISSUE_URL=$(echo "$FUZZING_ISSUES" | jq -r '.url' | head -1)
109+
if [ "$ISSUE_NUMBER" = "null" ] || [ -z "$ISSUE_NUMBER" ]; then
110+
echo "⚠️ Could not parse issue number from search results; will create a new issue"
111+
FUZZING_ISSUE_COUNT=0
112+
else
113+
echo "Adding comment to existing issue #$ISSUE_NUMBER"
114+
COMMENT_BODY="## Job Failed Again
115+
116+
**Date:** $(date '+%Y-%m-%d %H:%M:%S UTC')
117+
**Workflow Run:** [$WORKFLOW_RUN_URL]($WORKFLOW_RUN_URL)
118+
119+
The scheduled job has failed again. Please check the workflow logs and artifacts for details."
120+
121+
if gh issue comment "$ISSUE_NUMBER" --body "$COMMENT_BODY" --repo "$REPO"; then
122+
echo "✅ Added comment to existing issue #$ISSUE_NUMBER: $ISSUE_URL"
123+
exit 0
124+
else
125+
echo "❌ Failed to add comment to existing issue. Will attempt to create a new issue instead."
126+
FUZZING_ISSUE_COUNT=0
127+
fi
128+
fi
129+
fi
130+
131+
if [ "$FUZZING_ISSUE_COUNT" -eq 0 ]; then
132+
echo "No existing matching issues found. Creating a new issue..."
133+
134+
ISSUE_BODY="## Job Failure Report
135+
136+
**Date:** $(date '+%Y-%m-%d %H:%M:%S UTC')
137+
**Workflow Run:** [$WORKFLOW_RUN_URL]($WORKFLOW_RUN_URL)
138+
139+
### Details
140+
The scheduled job failed during execution. This issue was automatically created to track the failure. Please check the workflow logs and any uploaded artifacts for more details.
141+
142+
### Next Steps
143+
- [ ] Review the workflow logs for error details
144+
- [ ] Download and analyze any crash artifacts if available
145+
- [ ] Determine the root cause of the failure
146+
- [ ] Fix the underlying issue
147+
148+
---
149+
*This issue was automatically created by the CI failure notification system.*"
150+
151+
# Build label args for gh issue create
152+
LABEL_ARGS=()
153+
for lbl in "${LABEL_ARRAY[@]}"; do
154+
LABEL_ARGS+=("--label" "$lbl")
155+
done
156+
157+
if ISSUE_URL=$(gh issue create \
158+
--title "$ISSUE_TITLE" \
159+
--body "$ISSUE_BODY" \
160+
"${LABEL_ARGS[@]}" \
161+
--repo "$REPO"); then
162+
echo "✅ Created new issue: $ISSUE_URL"
163+
else
164+
echo "❌ Failed to create new issue"
165+
exit 1
166+
fi
167+
fi
168+
169+
echo "Notification script completed successfully"

‎dev/notify-fuzzing-failure.sh‎

Lines changed: 0 additions & 153 deletions
This file was deleted.

0 commit comments

Comments
 (0)