Skip to content

Commit 5374f8a

Browse files
Neon-WhiteNeon-White
authored
Merge pull request #9229 from Neon-White/nightly-ibm-cloud-manager
Add nightly IBM VM provision and cleanup tools
2 parents ec74a8b + 910439c commit 5374f8a

File tree

5 files changed

+362
-0
lines changed

5 files changed

+362
-0
lines changed

‎.github/workflows/ibm-daily-leftover-cleanup.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: Daily IBM leftover cleanup
2+
3+
on:
4+
schedule:
5+
- cron: '0 6 * * *' # Run daily at 6AM UTC
6+
workflow_dispatch: # Allow manual triggering for on-demand cleanup
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
cleanup-nb-nightly-resources:
13+
name: Clean up nb-nightly IBM resources
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 15
16+
defaults:
17+
run:
18+
shell: bash -euo pipefail {0}
19+
20+
steps:
21+
- name: Install IBM Cloud CLI + VPC plugin
22+
run: |
23+
curl -fsSL https://clis.cloud.ibm.com/install/linux | sh >/dev/null
24+
ibmcloud plugin install vpc-infrastructure -f >/dev/null
25+
ibmcloud --version
26+
27+
- name: Authenticate with IBM Cloud
28+
env:
29+
IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }}
30+
run: |
31+
ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" -r eu-de >/dev/null
32+
33+
- name: Clean up nb-nightly floating IPs
34+
run: |
35+
echo "🔍 Looking for floating IPs with 'nb-nightly' in the name"
36+
37+
# Get all floating IPs and filter by name containing 'nb-nightly'
38+
FLOATING_IPS=$(ibmcloud is floating-ips --output JSON 2>/dev/null || echo "[]")
39+
MATCHING_IPS=$(echo "${FLOATING_IPS}" | jq -r '.[] | select(.name | contains("nb-nightly")) | .id' 2>/dev/null || echo "")
40+
41+
if [ -n "${MATCHING_IPS}" ]
42+
then
43+
echo "🗑️ Releasing floating IP(s) with 'nb-nightly' in name"
44+
for IP_ID in ${MATCHING_IPS}
45+
do
46+
IP_NAME=$(echo "${FLOATING_IPS}" | jq -r ".[] | select(.id == \"${IP_ID}\") | .name")
47+
ibmcloud is floating-ip-release "${IP_ID}" --force >/dev/null
48+
echo "IP released successfully"
49+
done
50+
echo "✅ All nb-nightly floating IPs released successfully"
51+
else
52+
echo "ℹ️ No floating IPs found with 'nb-nightly' in name"
53+
fi
54+
55+
- name: Clean up nb-nightly IBM VMs
56+
run: |
57+
echo "🔍 Looking for IBM VMs with 'nb-nightly' in the name"
58+
59+
# Get all instances and filter by name containing 'nb-nightly'
60+
INSTANCES=$(ibmcloud is instances --output JSON 2>/dev/null || echo "[]")
61+
MATCHING_INSTANCES=$(echo "${INSTANCES}" | jq -r '.[] | select(.name | contains("nb-nightly")) | .id' 2>/dev/null || echo "")
62+
63+
if [ -z "${MATCHING_INSTANCES}" ]
64+
then
65+
echo "ℹ️ No VMs found with 'nb-nightly' in name"
66+
echo "This might be expected if the VMs were already deleted or never created."
67+
exit 0
68+
fi
69+
70+
echo "🔎 Found VM(s) with 'nb-nightly' in name"
71+
echo "🗑️ Deleting IBM VM(s)..."
72+
for INSTANCE_ID in ${MATCHING_INSTANCES}
73+
do
74+
INSTANCE_NAME=$(echo "${INSTANCES}" | jq -r ".[] | select(.id == \"${INSTANCE_ID}\") | .name")
75+
ibmcloud is instance-delete "${INSTANCE_ID}" --force >/dev/null
76+
echo "VM deleted successfully"
77+
done
78+
echo "✅ All nb-nightly IBM VMs deletion initiated"
79+
80+
# Wait up to 2 minutes for deletion to complete
81+
echo "⏳ Waiting for deletion to complete..."
82+
for i in {1..24}
83+
do
84+
sleep 5
85+
REMAINING_INSTANCES=$(ibmcloud is instances --output JSON 2>/dev/null | jq -r '.[] | select(.name | contains("nb-nightly")) | .id' 2>/dev/null || echo "")
86+
if [ -z "${REMAINING_INSTANCES}" ]
87+
then
88+
echo "✅ All nb-nightly VMs deleted successfully"
89+
break
90+
fi
91+
if [ $i -eq 24 ]
92+
then
93+
echo "⚠️ Some VMs may still be deleting (timeout reached)"
94+
REMAINING_COUNT=$(echo "${REMAINING_INSTANCES}" | wc -w)
95+
echo " ${REMAINING_COUNT} VM(s) still in deletion process"
96+
fi
97+
done

‎.github/workflows/ibm-nightly-cleanup-dispatcher.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
name: Nightly IBM VM Cleanup Dispatcher
2+
3+
on:
4+
schedule:
5+
# Run daily at 4 AM UTC (4 hours after the nightly provisioning at midnight)
6+
- cron: '0 4 * * *'
7+
workflow_dispatch: # Allow manual triggering for on-demand cleanup
8+
9+
jobs:
10+
cleanup-warp-vm:
11+
uses: ./.github/workflows/ibm-nightly-vm-cleanup.yaml
12+
secrets:
13+
VM_CONFIG: ${{ secrets.IBM_WARP_VM_CONFIG }}
14+
IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }}

‎.github/workflows/ibm-nightly-provision-dispatcher.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: Nightly IBM VM Provision Dispatcher
2+
3+
on:
4+
schedule:
5+
# Run daily at midnight UTC
6+
- cron: '0 0 * * *'
7+
workflow_dispatch: # Allow manual triggering for on-demand provisioning
8+
9+
jobs:
10+
provision-warp-vm:
11+
uses: ./.github/workflows/ibm-nightly-vm-provision.yaml
12+
with:
13+
VM_CLOUD_INIT_CONFIG_LOCATION: ".github/ibm-warp-runner-config.yaml"
14+
secrets:
15+
VM_CONFIG: ${{ secrets.IBM_WARP_VM_CONFIG }}
16+
IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }}
17+
IBM_COS_WRITER_CREDENTIALS: ${{ secrets.IBM_COS_WRITER_CREDENTIALS }}
18+
SLACK_NIGHTLY_RESULTS_URL: ${{ secrets.SLACK_NIGHTLY_RESULTS_URL }}

‎.github/workflows/ibm-nightly-vm-cleanup.yaml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: Nightly IBM VM Cleanup
2+
3+
on:
4+
workflow_call:
5+
secrets:
6+
VM_CONFIG:
7+
required: true
8+
IBM_CLOUD_API_KEY:
9+
required: true
10+
workflow_dispatch: # Allow manual triggering for on-demand cleanup
11+
12+
permissions:
13+
contents: read
14+
15+
jobs:
16+
cleanup-ibm-vm:
17+
name: Clean up IBM VM
18+
runs-on: ubuntu-latest
19+
timeout-minutes: 10
20+
defaults:
21+
run:
22+
shell: bash -euo pipefail {0}
23+
24+
steps:
25+
- name: Install IBM Cloud CLI + VPC plugin
26+
run: |
27+
curl -fsSL https://clis.cloud.ibm.com/install/linux | sh >/dev/null
28+
ibmcloud plugin install vpc-infrastructure -f >/dev/null
29+
ibmcloud --version
30+
31+
- name: Mask and export VM config from JSON secret
32+
env:
33+
VM_CONFIG: ${{ secrets.VM_CONFIG }}
34+
run: |
35+
# Mask non-empty values
36+
jq -r 'to_entries[] | select(.value != "") | "::add-mask::\(.value)"' <<<"${VM_CONFIG}"
37+
# Append KEY=VALUE lines to GITHUB_ENV
38+
jq -r 'to_entries[] | "\(.key)=\(.value)"' <<<"${VM_CONFIG}" >>"${GITHUB_ENV}"
39+
40+
41+
- name: Authenticate with IBM Cloud
42+
env:
43+
IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }}
44+
run: |
45+
ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" -r "${REGION}" >/dev/null
46+
47+
- name: Clean up floating IPs
48+
run: |
49+
echo "🔍 Looking for floating IPs with the resource tag"
50+
FLOATING_IP_IDS=$(ibmcloud resource search "tags:\"${RESOURCE_TAG}\" AND type:floating-ip" --output JSON | jq -r '.items[].resource_id' 2>/dev/null || echo "")
51+
52+
if [ -n "${FLOATING_IP_IDS}" ]
53+
then
54+
echo "🗑️ Releasing floating IP(s)"
55+
for IP_ID in ${FLOATING_IP_IDS}
56+
do
57+
ibmcloud is floating-ip-release "${IP_ID}" --force >/dev/null
58+
done
59+
echo "✅ All tagged floating IPs released successfully"
60+
else
61+
echo "ℹ️ No floating IPs found"
62+
fi
63+
64+
- name: Clean up IBM VMs
65+
run: |
66+
echo "🔍 Looking for IBM VMs with the resource tag"
67+
INSTANCE_IDS=$(ibmcloud resource search "tags:\"${RESOURCE_TAG}\" AND type:instance" --output JSON | jq -r '.items[].resource_id' 2>/dev/null || echo "")
68+
69+
if [ -z "${INSTANCE_IDS}" ]
70+
then
71+
echo "ℹ️ No VMs found with the resource tag"
72+
echo "This might be expected if the VMs were already deleted or never created."
73+
exit 0
74+
fi
75+
76+
echo "🔎 Found VM(s) with the resource tag"
77+
echo "🗑️ Deleting IBM VM(s)..."
78+
for INSTANCE_ID in ${INSTANCE_IDS}
79+
do
80+
ibmcloud is instance-delete "${INSTANCE_ID}" --force >/dev/null
81+
done
82+
echo "✅ All tagged IBM VMs deleted successfully"
83+
84+
# Wait up to 1 minute for deletion to complete
85+
for i in {1..12}
86+
do
87+
sleep 5
88+
REMAINING=$(ibmcloud resource search "tags:\"${RESOURCE_TAG}\" AND type:instance" --output JSON | jq -r '.items[].resource_id' 2>/dev/null || echo "")
89+
[ -z "${REMAINING}" ] && echo "✅ Deletion verified" && break
90+
[ $i -eq 12 ] && echo "⚠️ Some VMs may still be deleting"
91+
done

‎.github/workflows/ibm-nightly-vm-provision.yaml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: Nightly IBM VM Provisioning
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
VM_CLOUD_INIT_CONFIG_LOCATION:
7+
required: true
8+
type: string
9+
secrets:
10+
VM_CONFIG:
11+
required: true
12+
IBM_CLOUD_API_KEY:
13+
required: true
14+
IBM_COS_WRITER_CREDENTIALS:
15+
required: true
16+
SLACK_NIGHTLY_RESULTS_URL:
17+
required: true
18+
workflow_dispatch:
19+
20+
permissions:
21+
contents: read # required for actions/checkout
22+
23+
jobs:
24+
provision-ibm-vm:
25+
name: Provision IBM VM
26+
runs-on: ubuntu-latest
27+
timeout-minutes: 10
28+
defaults:
29+
run:
30+
shell: bash -euo pipefail {0}
31+
32+
steps:
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
36+
- name: Install IBM Cloud CLI + VPC plugin
37+
run: |
38+
curl -fsSL https://clis.cloud.ibm.com/install/linux | sh >/dev/null
39+
ibmcloud plugin install vpc-infrastructure -f >/dev/null
40+
ibmcloud --version
41+
42+
- name: Mask and export VM config from JSON secret
43+
env:
44+
VM_CONFIG: ${{ secrets.VM_CONFIG }}
45+
run: |
46+
# Mask non-empty values
47+
jq -r 'to_entries[] | select(.value != "") | "::add-mask::\(.value)"' <<<"${VM_CONFIG}"
48+
# Append KEY=VALUE lines to GITHUB_ENV
49+
jq -r 'to_entries[] | "\(.key)=\(.value)"' <<<"${VM_CONFIG}" >>"${GITHUB_ENV}"
50+
51+
52+
- name: Authenticate with IBM Cloud
53+
env:
54+
IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }}
55+
run: |
56+
ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" -r "${REGION}" >/dev/null
57+
58+
- name: Extract IBM COS credentials from JSON secret
59+
env:
60+
IBM_COS_WRITER_CREDENTIALS: ${{ secrets.IBM_COS_WRITER_CREDENTIALS }}
61+
run: |
62+
# Mask non-empty values
63+
jq -r 'to_entries[] | select(.value != "") | "::add-mask::\(.value)"' <<<"${IBM_COS_WRITER_CREDENTIALS}"
64+
# Append KEY=VALUE lines to GITHUB_ENV
65+
jq -r 'to_entries[] | "\(.key)=\(.value)"' <<<"${IBM_COS_WRITER_CREDENTIALS}" >>"${GITHUB_ENV}"
66+
67+
68+
- name: Create user-data with secrets
69+
env:
70+
SLACK_NIGHTLY_RESULTS_URL: ${{ secrets.SLACK_NIGHTLY_RESULTS_URL }}
71+
VM_CLOUD_INIT_CONFIG_LOCATION: ${{ inputs.VM_CLOUD_INIT_CONFIG_LOCATION }}
72+
run: |
73+
# Mask sensitive values
74+
[ -n "${SLACK_NIGHTLY_RESULTS_URL:-}" ] && echo "::add-mask::$SLACK_NIGHTLY_RESULTS_URL"
75+
export SLACK_NIGHTLY_RESULTS_URL
76+
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY WARP_LOGS_BUCKET IBM_COS_ENDPOINT
77+
78+
# Use envsubst to replace variables in the config
79+
envsubst < ${VM_CLOUD_INIT_CONFIG_LOCATION} > /tmp/ibm-vm-runner-config-with-secrets.yaml
80+
81+
- name: Provision IBM VM
82+
run: |
83+
TIMESTAMPED_INSTANCE_NAME="${INSTANCE_NAME}-$(date +"%Y-%m-%d-%H-%M-%S")"
84+
echo "::add-mask::$TIMESTAMPED_INSTANCE_NAME"
85+
echo "TIMESTAMPED_INSTANCE_NAME=$TIMESTAMPED_INSTANCE_NAME" >> "${GITHUB_ENV}"
86+
87+
if ! ibmcloud is instance-create \
88+
"${TIMESTAMPED_INSTANCE_NAME}" \
89+
"${VPC_NAME}" \
90+
"${ZONE}" \
91+
"${INSTANCE_PROFILE}" \
92+
"${SUBNET_ID}" \
93+
--image "${IMAGE_ID}" \
94+
--sgs "${SECURITY_GROUP_ID}" \
95+
--user-data @/tmp/ibm-vm-runner-config-with-secrets.yaml \
96+
> /dev/null 2>&1
97+
then
98+
echo "❌ Failed to provision IBM VM"
99+
exit 1
100+
fi
101+
echo "✅ IBM VM provisioned successfully"
102+
sleep 10
103+
104+
- name: Tag IBM VM
105+
run: |
106+
if ! ibmcloud resource tag-attach --tag-names "${RESOURCE_TAG}" --resource-name "${TIMESTAMPED_INSTANCE_NAME}" >/dev/null 2>&1
107+
then
108+
echo "❌ Failed to tag VM - cleanup may not work properly"
109+
exit 1
110+
fi
111+
echo "✅ IBM VM tagged successfully"
112+
113+
- name: Reserve and bind a floating IP
114+
run: |
115+
VM_JSON=$(ibmcloud is instance "${TIMESTAMPED_INSTANCE_NAME}" --output json)
116+
NIC_ID=$(echo "${VM_JSON}" | jq -r '.primary_network_attachment.virtual_network_interface.id')
117+
118+
TIMESTAMPED_FLOATING_IP_NAME="${FLOATING_IP_NAME}-$(date +"%Y-%m-%d-%H-%M-%S")"
119+
echo "::add-mask::$TIMESTAMPED_FLOATING_IP_NAME"
120+
echo "TIMESTAMPED_FLOATING_IP_NAME=$TIMESTAMPED_FLOATING_IP_NAME" >> "${GITHUB_ENV}"
121+
122+
# Reserve a floating IP in the same zone as the VM
123+
if ! ibmcloud is floating-ip-reserve \
124+
"${TIMESTAMPED_FLOATING_IP_NAME}" \
125+
--nic "${NIC_ID}" \
126+
> /dev/null 2>&1
127+
then
128+
echo "❌ Failed to reserve and bind floating IP"
129+
exit 1
130+
fi
131+
echo "✅ Floating IP reserved and bound successfully"
132+
sleep 10
133+
134+
- name: Tag floating IP
135+
run: |
136+
if ! ibmcloud resource tag-attach --tag-names "${RESOURCE_TAG}" --resource-name "${TIMESTAMPED_FLOATING_IP_NAME}" >/dev/null 2>&1
137+
then
138+
echo "❌ Failed to tag floating IP - cleanup may not work properly"
139+
# Consider releasing the floating IP here
140+
exit 1
141+
fi
142+
echo "✅ Floating IP tagged successfully"

0 commit comments

Comments
 (0)