Skip to content

Commit 345c4d9

Browse files
IllviljanIllviljan
committed
Try Iris benchmarks CI
1 parent 1546d93 commit 345c4d9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+7843
-0
lines changed

‎.github/workflows/benchmarks_report.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Post any reports generated by benchmarks_run.yml .
2+
# Separated for security:
3+
# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
4+
5+
name: benchmarks-report
6+
run-name: Report benchmark results
7+
8+
on:
9+
workflow_run:
10+
workflows: [benchmarks-run]
11+
types:
12+
- completed
13+
14+
jobs:
15+
download:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
reports_exist: ${{ steps.unzip.outputs.reports_exist }}
19+
steps:
20+
- name: Download artifact
21+
id: download-artifact
22+
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow
23+
uses: actions/github-script@v7
24+
with:
25+
script: |
26+
let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
27+
owner: context.repo.owner,
28+
repo: context.repo.repo,
29+
run_id: context.payload.workflow_run.id,
30+
});
31+
let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
32+
return artifact.name == "benchmark_reports"
33+
})[0];
34+
if (typeof matchArtifact != 'undefined') {
35+
let download = await github.rest.actions.downloadArtifact({
36+
owner: context.repo.owner,
37+
repo: context.repo.repo,
38+
artifact_id: matchArtifact.id,
39+
archive_format: 'zip',
40+
});
41+
let fs = require('fs');
42+
fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/benchmark_reports.zip`, Buffer.from(download.data));
43+
};
44+
45+
- name: Unzip artifact
46+
id: unzip
47+
run: |
48+
if test -f "benchmark_reports.zip"; then
49+
reports_exist=1
50+
unzip benchmark_reports.zip -d benchmark_reports
51+
else
52+
reports_exist=0
53+
fi
54+
echo "reports_exist=$reports_exist" >> "$GITHUB_OUTPUT"
55+
56+
- name: Store artifact
57+
uses: actions/upload-artifact@v4
58+
with:
59+
name: benchmark_reports
60+
path: benchmark_reports
61+
62+
post_reports:
63+
runs-on: ubuntu-latest
64+
needs: download
65+
if: needs.download.outputs.reports_exist == 1
66+
steps:
67+
- name: Checkout repo
68+
uses: actions/checkout@v5
69+
70+
- name: Download artifact
71+
uses: actions/download-artifact@v5
72+
with:
73+
name: benchmark_reports
74+
path: .github/workflows/benchmark_reports
75+
76+
- name: Set up Python
77+
# benchmarks/bm_runner.py only needs builtins to run.
78+
uses: actions/setup-python@v5
79+
80+
- name: Post reports
81+
env:
82+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
83+
run: benchmarks/bm_runner.py _gh_post

‎.github/workflows/benchmarks_run.yml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
# Use ASV to check for performance regressions, either:
2+
# - In the last 24 hours' commits.
3+
# - Introduced by this pull request.
4+
5+
name: benchmarks-run
6+
run-name: Run benchmarks
7+
8+
on:
9+
schedule:
10+
# Runs every day at 23:00.
11+
- cron: "0 23 * * *"
12+
workflow_dispatch:
13+
inputs:
14+
first_commit:
15+
description: "First commit to benchmark (see bm_runner.py > Overnight)."
16+
required: false
17+
type: string
18+
pull_request:
19+
# Add the `labeled` type to the default list.
20+
types: [labeled, opened, synchronize, reopened]
21+
22+
jobs:
23+
pre-checks:
24+
# This workflow supports two different scenarios (overnight and branch).
25+
# The pre-checks job determines which scenario is being run.
26+
runs-on: ubuntu-latest
27+
if: github.repository == 'SciTools/iris'
28+
outputs:
29+
overnight: ${{ steps.overnight.outputs.check }}
30+
branch: ${{ steps.branch.outputs.check }}
31+
steps:
32+
- uses: actions/checkout@v5
33+
with:
34+
fetch-depth: 2
35+
- id: files-changed
36+
uses: marceloprado/has-changed-path@df1b7a3161b8fb9fd8c90403c66a9e66dfde50cb
37+
with:
38+
# SEE ALSO .github/labeler.yml .
39+
paths: requirements/locks/*.lock
40+
- id: overnight
41+
name: Check overnight scenario
42+
if: github.event_name != 'pull_request'
43+
run: echo "check=true" >> "$GITHUB_OUTPUT"
44+
- id: branch
45+
name: Check branch scenario
46+
if: >
47+
github.event_name == 'pull_request'
48+
&&
49+
(
50+
steps.files-changed.outputs.changed == 'true'
51+
||
52+
github.event.label.name == 'benchmark_this'
53+
)
54+
run: echo "check=true" >> "$GITHUB_OUTPUT"
55+
56+
57+
benchmark:
58+
runs-on: ubuntu-latest
59+
needs: pre-checks
60+
if: >
61+
needs.pre-checks.outputs.overnight == 'true' ||
62+
needs.pre-checks.outputs.branch == 'true'
63+
64+
env:
65+
IRIS_TEST_DATA_LOC_PATH: benchmarks
66+
IRIS_TEST_DATA_PATH: benchmarks/iris-test-data
67+
IRIS_TEST_DATA_VERSION: "2.28"
68+
# Lets us manually bump the cache to rebuild
69+
ENV_CACHE_BUILD: "0"
70+
TEST_DATA_CACHE_BUILD: "2"
71+
72+
steps:
73+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
74+
- name: Checkout repo
75+
uses: actions/checkout@v5
76+
with:
77+
fetch-depth: 0
78+
79+
- name: Install run dependencies
80+
run: pip install asv nox!=2025.05.01
81+
82+
- name: Cache environment directories
83+
id: cache-env-dir
84+
uses: actions/cache@v4
85+
with:
86+
path: |
87+
.nox
88+
benchmarks/.asv/env
89+
$CONDA/pkgs
90+
key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }}
91+
92+
- name: Cache test data directory
93+
id: cache-test-data
94+
uses: actions/cache@v4
95+
with:
96+
path: |
97+
${{ env.IRIS_TEST_DATA_PATH }}
98+
key:
99+
test-data-${{ env.IRIS_TEST_DATA_VERSION }}-${{ env.TEST_DATA_CACHE_BUILD }}
100+
101+
- name: Fetch the test data
102+
if: steps.cache-test-data.outputs.cache-hit != 'true'
103+
run: |
104+
wget --quiet https://github.com/SciTools/iris-test-data/archive/v${IRIS_TEST_DATA_VERSION}.zip -O iris-test-data.zip
105+
unzip -q iris-test-data.zip
106+
mkdir --parents ${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_LOC_PATH}
107+
mv iris-test-data-${IRIS_TEST_DATA_VERSION} ${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH}
108+
109+
- name: Set test data var
110+
run: |
111+
echo "OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH}/test_data" >> $GITHUB_ENV
112+
113+
- name: Benchmark this pull request
114+
# If the 'branch' condition(s) are met: use the bm_runner to compare
115+
# the proposed merge with the base branch.
116+
if: needs.pre-checks.outputs.branch == 'true'
117+
env:
118+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
119+
PR_NUMBER: ${{ github.event.number }}
120+
run: |
121+
nox -s benchmarks -- branch origin/${{ github.base_ref }}
122+
123+
- name: Run overnight benchmarks
124+
# If the 'overnight' condition(s) are met: use the bm_runner to compare
125+
# each of the last 24 hours' commits to their parents.
126+
id: overnight
127+
if: needs.pre-checks.outputs.overnight == 'true'
128+
env:
129+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
130+
# The first_commit argument allows a custom starting point - useful
131+
# for manual re-running.
132+
run: |
133+
first_commit=${{ inputs.first_commit }}
134+
if [ "$first_commit" == "" ]
135+
then
136+
first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1)
137+
fi
138+
139+
if [ "$first_commit" != "" ]
140+
then
141+
nox -s benchmarks -- overnight $first_commit
142+
fi
143+
144+
- name: Warn of failure
145+
# The overnight run is not on a pull request, so a failure could go
146+
# unnoticed without being actively advertised.
147+
if: >
148+
failure() &&
149+
steps.overnight.outcome == 'failure'
150+
env:
151+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
152+
run: |
153+
title="Overnight benchmark workflow failed: \`${{ github.run_id }}\`"
154+
body="Generated by GHA run [\`${{github.run_id}}\`](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})"
155+
gh issue create --title "$title" --body "$body" --label "Bot" --label "Type: Performance" --repo $GITHUB_REPOSITORY
156+
157+
- name: Upload any benchmark reports
158+
# Uploading enables more downstream processing e.g. posting a PR comment.
159+
if: success() || steps.overnight.outcome == 'failure'
160+
uses: actions/upload-artifact@v4
161+
with:
162+
name: benchmark_reports
163+
path: .github/workflows/benchmark_reports
164+
165+
- name: Archive asv results
166+
# Store the raw ASV database(s) to help manual investigations.
167+
if: ${{ always() }}
168+
uses: actions/upload-artifact@v4
169+
with:
170+
name: asv-raw-results
171+
path: benchmarks/.asv/results

‎.github/workflows/benchmarks_validate.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: benchmarks-validate
2+
run-name: Validate the benchmarking setup
3+
4+
on:
5+
push:
6+
branches:
7+
- "main"
8+
- "v*x"
9+
tags:
10+
- "v*"
11+
pull_request:
12+
branches:
13+
- "*"
14+
workflow_dispatch:
15+
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.ref }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
validate:
22+
runs-on: ubuntu-latest
23+
24+
env:
25+
# Lets us manually bump the cache to rebuild
26+
ENV_CACHE_BUILD: "0"
27+
28+
steps:
29+
- name: Checkout repo
30+
uses: actions/checkout@v5
31+
with:
32+
fetch-depth: 0
33+
34+
- name: Install run dependencies
35+
run: pip install asv nox!=2025.05.01
36+
37+
- name: Cache environment directories
38+
id: cache-env-dir
39+
uses: actions/cache@v4
40+
with:
41+
path: |
42+
.nox
43+
benchmarks/.asv/env
44+
$CONDA/pkgs
45+
key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }}
46+
47+
- name: Validate setup
48+
run: nox -s benchmarks -- validate

0 commit comments

Comments
 (0)