Skip to content

Commit e266a4e

Browse files
committed
fix: avoid cache busting through headers in github api calls
1 parent d80f1e2 commit e266a4e

File tree

5 files changed

+126
-28
lines changed

5 files changed

+126
-28
lines changed

.ci/resolve_git_ref

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/bin/sh
2+
# Resolve git refs (branches, tags, or commit SHAs) to full commit SHAs
3+
#
4+
# Usage:
5+
# resolve_git_ref <repo> <ref>
6+
# resolve_git_ref eic/edm4eic main
7+
# resolve_git_ref eic/epic v24.11.0
8+
# resolve_git_ref eic/eicrecon abc123def456...
9+
#
10+
# Exit codes:
11+
# 0 - Success (ref resolved or already a SHA)
12+
# 1 - Invalid arguments or ref not found
13+
14+
set -e
15+
16+
# Show usage if no arguments or --help
17+
if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
18+
cat >&2 << 'USAGE'
19+
Usage: resolve_git_ref <repo> <ref>
20+
21+
Resolves a git reference (branch, tag, or commit SHA) to a full commit SHA.
22+
23+
Arguments:
24+
repo Git repository, in the format org/repo (e.g., eic/edm4eic)
25+
ref Git reference: branch name, tag name, or commit SHA
26+
27+
Examples:
28+
# Resolve branch to SHA
29+
resolve_git_ref eic/edm4eic main
30+
31+
# Resolve tag to SHA
32+
resolve_git_ref eic/epic v24.11.0
33+
34+
# Return commit SHA as-is
35+
resolve_git_ref eic/eicrecon abc123def456...
36+
37+
Exit codes:
38+
0 - Success (SHA printed to stdout)
39+
1 - Error (message printed to stderr)
40+
USAGE
41+
exit 1
42+
fi
43+
44+
# Require exactly 2 arguments
45+
if [ $# -ne 2 ]; then
46+
echo "Error: Expected 2 arguments, got $#" >&2
47+
echo "Run 'resolve_git_ref --help' for usage" >&2
48+
exit 1
49+
fi
50+
51+
repo_url="https://github.com/${1}.git"
52+
ref="${2}"
53+
54+
# Handle empty ref
55+
if [ -z "${ref}" ]; then
56+
echo "Error: ref argument is empty" >&2
57+
exit 1
58+
fi
59+
60+
# If it's already a commit SHA (7-40 hex characters), return as-is
61+
# Full SHAs are 40 chars, but short SHAs (7+) are also valid
62+
# POSIX-compliant check: use case statement for pattern matching
63+
case "${ref}" in
64+
*[!0-9a-f]*)
65+
# Contains non-hex characters, not a SHA
66+
;;
67+
*)
68+
# All hex characters, check length
69+
ref_len=${#ref}
70+
if [ "${ref_len}" -ge 7 ] && [ "${ref_len}" -le 40 ]; then
71+
echo "${ref}"
72+
exit 0
73+
fi
74+
;;
75+
esac
76+
77+
# Try to resolve as branch
78+
sha=$(git ls-remote "${repo_url}" "refs/heads/${ref}" 2>/dev/null | cut -f1)
79+
if [ -n "${sha}" ]; then
80+
echo "${sha}"
81+
exit 0
82+
fi
83+
84+
# Try to resolve as tag
85+
sha=$(git ls-remote "${repo_url}" "refs/tags/${ref}" 2>/dev/null | cut -f1)
86+
if [ -n "${sha}" ]; then
87+
echo "${sha}"
88+
exit 0
89+
fi
90+
91+
# Try to resolve as annotated tag (^{} dereferences to commit)
92+
sha=$(git ls-remote "${repo_url}" "refs/tags/${ref}^{}" 2>/dev/null | cut -f1)
93+
if [ -n "${sha}" ]; then
94+
echo "${sha}"
95+
exit 0
96+
fi
97+
98+
# Unable to resolve
99+
echo "Error: Unable to resolve ref '${ref}' in repository '${repo_url}'" >&2
100+
echo "Ref must be a valid branch name, tag name, or commit SHA" >&2
101+
exit 1

.github/workflows/build-push.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ jobs:
8282
run: |
8383
source spack.sh
8484
echo "orgrepo=${SPACK_ORGREPO}" | tee -a $GITHUB_OUTPUT
85-
echo "version=${SPACK_VERSION}" | tee -a $GITHUB_OUTPUT
85+
echo "version=$(.ci/resolve_git_ref ${SPACK_ORGREPO} ${SPACK_VERSION})" | tee -a $GITHUB_OUTPUT
8686
echo "cherrypicks=${SPACK_CHERRYPICKS//$'\n'/ }" | tee -a $GITHUB_OUTPUT
8787
echo "cherrypicks_files=${SPACK_CHERRYPICKS_FILES//$'\n'/ }" | tee -a $GITHUB_OUTPUT
8888
- name: Load spack-packages version and cherry-picks
@@ -91,21 +91,21 @@ jobs:
9191
run: |
9292
source spack-packages.sh
9393
echo "orgrepo=${SPACKPACKAGES_ORGREPO}" | tee -a $GITHUB_OUTPUT
94-
echo "version=${SPACKPACKAGES_VERSION}" | tee -a $GITHUB_OUTPUT
94+
echo "version=$(.ci/resolve_git_ref ${SPACKPACKAGES_ORGREPO} ${SPACKPACKAGES_VERSION})" | tee -a $GITHUB_OUTPUT
9595
echo "cherrypicks=${SPACKPACKAGES_CHERRYPICKS//$'\n'/ }" | tee -a $GITHUB_OUTPUT
9696
echo "cherrypicks_files=${SPACKPACKAGES_CHERRYPICKS_FILES//$'\n'/ }" | tee -a $GITHUB_OUTPUT
9797
- name: Load key4hep-spack version
9898
id: key4hep-spack
9999
run: |
100100
source key4hep-spack.sh
101101
echo "orgrepo=${KEY4HEPSPACK_ORGREPO}" | tee -a $GITHUB_OUTPUT
102-
echo "version=${KEY4HEPSPACK_VERSION}" | tee -a $GITHUB_OUTPUT
102+
echo "version=$(.ci/resolve_git_ref ${KEY4HEPSPACK_ORGREPO} ${KEY4HEPSPACK_VERSION})" | tee -a $GITHUB_OUTPUT
103103
- name: Load eic-spack version
104104
id: eic-spack
105105
run: |
106106
source eic-spack.sh
107107
echo "orgrepo=${EICSPACK_ORGREPO}" | tee -a $GITHUB_OUTPUT
108-
echo "version=${EICSPACK_VERSION}" | tee -a $GITHUB_OUTPUT
108+
echo "version=$(.ci/resolve_git_ref ${EICSPACK_ORGREPO} ${EICSPACK_VERSION})" | tee -a $GITHUB_OUTPUT
109109
- name: Set up QEMU
110110
uses: docker/setup-qemu-action@v3
111111
with:

.gitlab-ci.yml

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -315,17 +315,17 @@ base:
315315
--build-arg BASE_IMAGE=${BASE_IMAGE}
316316
--build-arg BUILD_IMAGE=${BUILD_IMAGE}
317317
--build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
318-
--build-arg SPACK_VERSION=${SPACK_VERSION}
318+
--build-arg SPACK_VERSION=$(sh .ci/resolve_git_ref "${SPACK_ORGREPO}" "${SPACK_VERSION}")
319319
--build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
320320
--build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
321321
--build-arg SPACKPACKAGES_ORGREPO=${SPACKPACKAGES_ORGREPO}
322-
--build-arg SPACKPACKAGES_VERSION=${SPACKPACKAGES_VERSION}
322+
--build-arg SPACKPACKAGES_VERSION=$(sh .ci/resolve_git_ref "${SPACKPACKAGES_ORGREPO}" "${SPACKPACKAGES_VERSION}")
323323
--build-arg SPACKPACKAGES_CHERRYPICKS="${SPACKPACKAGES_CHERRYPICKS}"
324324
--build-arg SPACKPACKAGES_CHERRYPICKS_FILES="${SPACKPACKAGES_CHERRYPICKS_FILES}"
325325
--build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}
326-
--build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}
326+
--build-arg KEY4HEPSPACK_VERSION=$(sh .ci/resolve_git_ref "${KEY4HEPSPACK_ORGREPO}" "${KEY4HEPSPACK_VERSION}")
327327
--build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
328-
--build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
328+
--build-arg EICSPACK_VERSION=$(sh .ci/resolve_git_ref "${EICSPACK_ORGREPO}" "${EICSPACK_VERSION}")
329329
--build-arg jobs=${JOBS}
330330
--provenance false
331331
containers/debian
@@ -449,16 +449,16 @@ eic:
449449
--build-arg EIC_CONTAINER_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
450450
--build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
451451
${IF_BUILD_DEFAULT+
452-
${EDM4EIC_VERSION:+--build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION}}
453-
${EICRECON_VERSION:+--build-arg EICRECON_VERSION=${EICRECON_VERSION}}
454-
${EPIC_VERSION:+--build-arg EPIC_VERSION=${EPIC_VERSION}}
455-
${JUGGLER_VERSION:+--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}}
452+
${EDM4EIC_VERSION:+--build-arg EDM4EIC_VERSION=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION})}
453+
${EICRECON_VERSION:+--build-arg EICRECON_VERSION=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION})}
454+
${EPIC_VERSION:+--build-arg EPIC_VERSION=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION})}
455+
${JUGGLER_VERSION:+--build-arg JUGGLER_VERSION=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION})}
456456
}
457457
${IF_BUILD_NIGHTLY+
458-
--build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION:-main}
459-
--build-arg EICRECON_VERSION=${EICRECON_VERSION:-main}
460-
--build-arg EPIC_VERSION=${EPIC_VERSION:-main}
461-
--build-arg JUGGLER_VERSION=${JUGGLER_VERSION:-main}
458+
--build-arg EDM4EIC_VERSION=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION:-main})
459+
--build-arg EICRECON_VERSION=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION:-main})
460+
--build-arg EPIC_VERSION=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION:-main})
461+
--build-arg JUGGLER_VERSION=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main})
462462
}
463463
--build-arg ENV=${ENV}
464464
--build-arg jobs=${JOBS}

containers/debian/Dockerfile

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,11 @@ ARG SPACK_VERSION="develop"
160160
ENV SPACK_PYTHON=/usr/bin/python3
161161
ARG SPACK_CHERRYPICKS=""
162162
ARG SPACK_CHERRYPICKS_FILES=""
163-
ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
163+
# Dockerfile ADD --keep-git-dir clones a shallow copy preventing cherry-picking
164+
#ADD --keep-git-dir https://github.com/${SPACK_ORGREPO}.git#${SPACK_VERSION} ${SPACK_ROOT}
165+
# so we git clone the default branch and checkout the specific commit
164166
RUN <<EOF
167+
set -e
165168
git clone --filter=tree:0 https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}
166169
git -C ${SPACK_ROOT} checkout ${SPACK_VERSION}
167170
if [ -n "${SPACK_CHERRYPICKS}" ] ; then
@@ -194,7 +197,9 @@ ARG SPACKPACKAGES_ORGREPO="spack/spack-packages"
194197
ARG SPACKPACKAGES_VERSION="develop"
195198
ARG SPACKPACKAGES_CHERRYPICKS=""
196199
ARG SPACKPACKAGES_CHERRYPICKS_FILES=""
197-
ADD https://api.github.com/repos/${SPACKPACKAGES_ORGREPO}/commits/${SPACKPACKAGES_VERSION} /tmp/spack-packages.json
200+
# Dockerfile ADD --keep-git-dir clones a shallow copy preventing cherry-picking
201+
#ADD --keep-git-dir https://github.com/${SPACKPACKAGES_ORGREPO}.git#${SPACKPACKAGES_VERSION} ${SPACKPACKAGES_ROOT}
202+
# so we git clone the default branch and checkout the specific commit
198203
RUN <<EOF
199204
set -e
200205
git clone --filter=tree:0 https://github.com/${SPACKPACKAGES_ORGREPO}.git ${SPACKPACKAGES_ROOT}
@@ -247,14 +252,14 @@ spack mirror add --scope site --unsigned ghcr-${SPACK_VERSION} oci://ghcr.io/eic
247252
spack mirror list
248253
EOF
249254

250-
## Setup key4hep-spack
255+
## Setup key4hep-spack (no need for cherry-picks)
251256
ENV KEY4HEPSPACK_ROOT=${SPACKPACKAGES_ROOT}/repos/key4hep-spack
252257
ARG KEY4HEPSPACK_ORGREPO="key4hep/key4hep-spack"
253258
ARG KEY4HEPSPACK_VERSION="main"
254259
ADD https://github.com/${KEY4HEPSPACK_ORGREPO}.git#${KEY4HEPSPACK_VERSION} ${KEY4HEPSPACK_ROOT}
255260
RUN spack repo add --scope site "${KEY4HEPSPACK_ROOT}"
256261

257-
## Setup eic-spack
262+
## Setup eic-spack (no need for cherry-picks)
258263
ENV EICSPACK_ROOT=${SPACKPACKAGES_ROOT}/repos/eic-spack
259264
ARG EICSPACK_ORGREPO="eic/eic-spack"
260265
ARG EICSPACK_VERSION="develop"

containers/eic/Dockerfile

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -165,33 +165,25 @@ ARG EDM4EIC_VERSION="8aeb507f93a93257c99985efbce0ec1371e0b331"
165165
ARG EICRECON_VERSION="28108da4a1e8919a05dfdb5f11e114800a2cbe96"
166166
ARG EPIC_VERSION="c1827f05430b2051df8a0b421db1cbab87165e0b"
167167
ARG JUGGLER_VERSION="df87bf1f8643afa8e80bece9d36d6dc26dfe8132"
168-
ADD https://api.github.com/repos/eic/edm4eic/commits/${EDM4EIC_VERSION} /tmp/edm4eic.json
169-
ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
170-
ADD https://api.github.com/repos/eic/epic/commits/${EPIC_VERSION} /tmp/epic.json
171-
ADD https://api.github.com/repos/eic/juggler/commits/${JUGGLER_VERSION} /tmp/juggler.json
172168

173169
# Concretization (custom environment)
174170
RUN <<EOF
175171
set -e
176172
spack env activate --dir ${SPACK_ENV}
177173
if [ "${EDM4EIC_VERSION}" != "8aeb507f93a93257c99985efbce0ec1371e0b331" ] ; then
178-
export EDM4EIC_VERSION=$(jq -r .sha /tmp/edm4eic.json)
179174
sed -i "/# EDM4EIC_VERSION$/ s/@[^' ]*/@git.${EDM4EIC_VERSION}=main/" /opt/spack-environment/packages.yaml
180175
spack deconcretize -y --all edm4eic
181176
fi
182177
if [ "${EICRECON_VERSION}" != "28108da4a1e8919a05dfdb5f11e114800a2cbe96" ] ; then
183-
export EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
184178
sed -i "/# EICRECON_VERSION$/ s/@[^' ]*/@git.${EICRECON_VERSION}=main/" /opt/spack-environment/packages.yaml
185179
spack deconcretize -y --all eicrecon
186180
fi
187181
if [ "${EPIC_VERSION}" != "c1827f05430b2051df8a0b421db1cbab87165e0b" ] ; then
188-
export EPIC_VERSION=$(jq -r .sha /tmp/epic.json)
189182
sed -i "/# EPIC_VERSION$/ s/epic\s/epic@git.${EPIC_VERSION}=main /" /opt/spack-environment/${ENV}/spack.yaml
190183
sed -i "/# EPIC_VERSION$/ s/epic@main\s/epic@git.${EPIC_VERSION}=main /" /opt/spack-environment/${ENV}/spack.yaml
191184
spack deconcretize -y --all epic
192185
fi
193186
if [ "${JUGGLER_VERSION}" != "df87bf1f8643afa8e80bece9d36d6dc26dfe8132" ] ; then
194-
export JUGGLER_VERSION=$(jq -r .sha /tmp/juggler.json)
195187
sed -i "/# JUGGLER_VERSION$/ s/@[^' ]*/@git.${JUGGLER_VERSION}=main/" /opt/spack-environment/packages.yaml
196188
spack deconcretize -y --all juggler
197189
fi

0 commit comments

Comments
 (0)