From 253bd4aec18520e3a73a3be0913d0e4cb69b3dde Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 14 Aug 2025 11:11:42 -0500 Subject: [PATCH 01/44] feat: re-enable arm64 builds, per ROOT team request --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a2caec6..935ffb86 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -273,7 +273,7 @@ base: # PLATFORM: linux/amd64,linux/arm64/v8 - BASE_IMAGE: debian:trixie-slim BUILD_IMAGE: debian_stable_base - PLATFORM: linux/amd64 + PLATFORM: linux/amd64,linux/arm64/v8 # - BASE_IMAGE: ubuntu:22.04 # BUILD_IMAGE: ubuntu_base # PLATFORM: linux/amd64,linux/arm64/v8 @@ -367,7 +367,7 @@ eic: - nightly BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64 + PLATFORM: linux/amd64,linux/arm64/v8 # - BUILD_IMAGE: oneapi_ # ENV: # - prod From 80d71e6665dfb8bb93e61019893fdcfb96a0fbdb Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 14 Aug 2025 11:13:37 -0500 Subject: [PATCH 02/44] fix: add linux/arm64/v8 to dependent jobs needs --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 935ffb86..4cdbb139 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -581,7 +581,7 @@ eic_xl:singularity:default: - default BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64 + PLATFORM: linux/amd64,linux/arm64/v8 # eic-shell/install.py expects build/eic_xl.sif from the job with name 'eic_xl:singularity:nightly' eic_xl:singularity:nightly: @@ -601,7 +601,7 @@ eic_xl:singularity:nightly: - nightly BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64 + PLATFORM: linux/amd64,linux/arm64/v8 .benchmarks: stage: benchmarks @@ -630,7 +630,7 @@ eic_xl:singularity:nightly: - default BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64 + PLATFORM: linux/amd64,linux/arm64/v8 .benchmarks:nightly: extends: .benchmarks @@ -646,7 +646,7 @@ eic_xl:singularity:nightly: - nightly BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64 + PLATFORM: linux/amd64,linux/arm64/v8 benchmarks:geoviewer:default: extends: .benchmarks:default From 80065357694f940d29ed3a8463df84199c064d1b Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 10:01:35 -0500 Subject: [PATCH 03/44] feat: parallelize amd64 and arm64; merge in manifest --- .gitlab-ci.yml | 62 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4cdbb139..f4bf17b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -273,7 +273,9 @@ base: # PLATFORM: linux/amd64,linux/arm64/v8 - BASE_IMAGE: debian:trixie-slim BUILD_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 + PLATFORM: + - linux/amd64 + - linux/arm64/v8 # - BASE_IMAGE: ubuntu:22.04 # BUILD_IMAGE: ubuntu_base # PLATFORM: linux/amd64,linux/arm64/v8 @@ -304,12 +306,7 @@ base: --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64 --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64 --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max - --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} - ${EXPORT_TAG:+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - } + ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE} --file containers/debian/Dockerfile --platform ${PLATFORM} --build-arg BASE_IMAGE=${BASE_IMAGE} @@ -328,6 +325,7 @@ base: --build-arg EICSPACK_VERSION=$(sh .ci/resolve_git_ref "${EICSPACK_ORGREPO}" "${EICSPACK_VERSION}") --build-arg jobs=${JOBS} --provenance false + --output push-by-digest=true,type=image,push=true containers/debian 2>&1 | tee build.log ; do @@ -343,6 +341,44 @@ base: let attempts=$attempts+1 ; done +base:multi: + parallel: + matrix: +# - BASE_IMAGE: debian:testing-slim +# BUILD_IMAGE: debian_testing_base +# PLATFORM: linux/amd64,linux/arm64/v8 + - BASE_IMAGE: debian:trixie-slim + BUILD_IMAGE: debian_stable_base + PLATFORM: linux/amd64,linux/arm64/v8 +# - BASE_IMAGE: ubuntu:22.04 +# BUILD_IMAGE: ubuntu_base +# PLATFORM: linux/amd64,linux/arm64/v8 +# - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04 +# BUILD_IMAGE: oneapi_base +# PLATFORM: linux/amd64 + - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-devel-${CUDA_OS} + BUILD_IMAGE: cuda_devel + PLATFORM: linux/amd64 + - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-runtime-${CUDA_OS} + BUILD_IMAGE: cuda_runtime + PLATFORM: linux/amd64 + extends: .build + stage: base + needs: base + script: + - declare -A amends ; + for platform in ${PLATFORM} ; do + amends[${platform}]="--amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@sha256:${digest}" ; + done + - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends[@]} + - for export_tag in ${EXPORT_TAG:+ + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} + } ; do + docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${export_tag} ; + docker push ${export_tag} ; + done eic: parallel: @@ -367,7 +403,9 @@ eic: - nightly BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 + PLATFORM: + - linux/amd64 + - linux/arm64/v8 # - BUILD_IMAGE: oneapi_ # ENV: # - prod @@ -581,7 +619,7 @@ eic_xl:singularity:default: - default BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 + PLATFORM: linux/amd64 # eic-shell/install.py expects build/eic_xl.sif from the job with name 'eic_xl:singularity:nightly' eic_xl:singularity:nightly: @@ -601,7 +639,7 @@ eic_xl:singularity:nightly: - nightly BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 + PLATFORM: linux/amd64 .benchmarks: stage: benchmarks @@ -630,7 +668,7 @@ eic_xl:singularity:nightly: - default BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 + PLATFORM: linux/amd64 .benchmarks:nightly: extends: .benchmarks @@ -646,7 +684,7 @@ eic_xl:singularity:nightly: - nightly BUILDER_IMAGE: debian_stable_base RUNTIME_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 + PLATFORM: linux/amd64 benchmarks:geoviewer:default: extends: .benchmarks:default From 5e70e67d3f5139c7474197aaf9bdfe5d4fb2fa50 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 10:02:39 -0500 Subject: [PATCH 04/44] fix: jobs:base:multi:needs as array --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f4bf17b3..cbec8223 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -364,7 +364,8 @@ base:multi: PLATFORM: linux/amd64 extends: .build stage: base - needs: base + needs: + - base script: - declare -A amends ; for platform in ${PLATFORM} ; do From 4879d1e1ac01621e19e0803040875c5a8edbbfae Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 10:05:39 -0500 Subject: [PATCH 05/44] fix: docker buildx build --tag --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbec8223..79985114 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -306,7 +306,7 @@ base: --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64 --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64 --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max - ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE} + --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE} --file containers/debian/Dockerfile --platform ${PLATFORM} --build-arg BASE_IMAGE=${BASE_IMAGE} From 7e0f213169ea816e6acbd4cee5de7aa8b17eb007 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 10:34:32 -0500 Subject: [PATCH 06/44] feat: store build metadata json --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79985114..638d03a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -244,6 +244,7 @@ status:pending: artifacts: paths: - build.log + - metadata/ when: always ## rules for nightly jobs @@ -326,6 +327,7 @@ base: --build-arg jobs=${JOBS} --provenance false --output push-by-digest=true,type=image,push=true + --metadata-file metadata/${BUILD_IMAGE}/${PLATFORM}/build.json containers/debian 2>&1 | tee build.log ; do From f0a70d1fb8d9d6a4280981ee5432817d5831f3c3 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 10:43:00 -0500 Subject: [PATCH 07/44] fix: avoid declare and append to var instead --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 638d03a5..413ae928 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -369,11 +369,12 @@ base:multi: needs: - base script: - - declare -A amends ; - for platform in ${PLATFORM} ; do - amends[${platform}]="--amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@sha256:${digest}" ; + - for platform in ${PLATFORM} ; do + digest=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${platform}/build.json) ; + echo ${digest} ; + amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; done - - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends[@]} + - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends} - for export_tag in ${EXPORT_TAG:+ ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} From 95af267a415ceec9d127e2f838c2484832946ef6 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 17:27:51 -0500 Subject: [PATCH 08/44] fix: mkdir metadata dir --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 413ae928..e15e1be0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -298,6 +298,7 @@ base: source spack-packages.sh ; source key4hep-spack.sh ; source eic-spack.sh ; + - mkdir -p metadata/${BUILD_IMAGE}/${PLATFORM} - attempts=0 - nocache="" - while ! From 7a5182319db2f8c8269dadad208244cc1b7c92c5 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 15 Aug 2025 18:07:08 -0500 Subject: [PATCH 09/44] fix: fun with anchors --- .gitlab-ci.yml | 96 +++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e15e1be0..a4cac8c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -266,7 +266,7 @@ status:pending: ## TODO ## oneapi_runtime + eic_xl --> oneapi_prod -base: +.base: &base_parallel parallel: matrix: # - BASE_IMAGE: debian:testing-slim @@ -289,6 +289,9 @@ base: - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-runtime-${CUDA_OS} BUILD_IMAGE: cuda_runtime PLATFORM: linux/amd64 + +base: + <<: *base_parallel extends: .build stage: base needs: @@ -345,33 +348,14 @@ base: done base:multi: - parallel: - matrix: -# - BASE_IMAGE: debian:testing-slim -# BUILD_IMAGE: debian_testing_base -# PLATFORM: linux/amd64,linux/arm64/v8 - - BASE_IMAGE: debian:trixie-slim - BUILD_IMAGE: debian_stable_base - PLATFORM: linux/amd64,linux/arm64/v8 -# - BASE_IMAGE: ubuntu:22.04 -# BUILD_IMAGE: ubuntu_base -# PLATFORM: linux/amd64,linux/arm64/v8 -# - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04 -# BUILD_IMAGE: oneapi_base -# PLATFORM: linux/amd64 - - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-devel-${CUDA_OS} - BUILD_IMAGE: cuda_devel - PLATFORM: linux/amd64 - - BASE_IMAGE: nvidia/cuda:${CUDA_VERSION}-runtime-${CUDA_OS} - BUILD_IMAGE: cuda_runtime - PLATFORM: linux/amd64 + <<: *base_parallel extends: .build stage: base needs: - base script: - - for platform in ${PLATFORM} ; do - digest=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${platform}/build.json) ; + - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do + digest=$(jq -r '."containerimage.digest"' ${build_json}) ; echo ${digest} ; amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; done @@ -385,7 +369,7 @@ base:multi: docker push ${export_tag} ; done -eic: +.eic: &eic_parallel parallel: matrix: - BUILD_IMAGE: eic_ @@ -443,6 +427,9 @@ eic: - if: '$ENV != "ci" && $CI_PIPELINE_SOURCE == "trigger"' when: manual - when: always + +eic: + <<: *eic_parallel extends: .build stage: eic needs: @@ -468,26 +455,7 @@ eic: --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_DEFAULT_BRANCH_SLUG}-amd64 --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_DEFAULT_BRANCH_SLUG}-amd64 --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${BUILD_TYPE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max - --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} - ${EXPORT_TAG:+ - ${IF_BUILD_DEFAULT+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - } - ${IF_BUILD_DEFAULT- - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - } - } - ${IF_BUILD_NIGHTLY+ - ${NIGHTLY:+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - } - } + --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV} --file containers/eic/Dockerfile --platform ${PLATFORM} --build-arg BENCHMARK_COM_VERSION=$(sh .ci/resolve_git_ref https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git master) @@ -524,6 +492,8 @@ eic: --secret type=env,id=GITHUB_REGISTRY_USER,env=GITHUB_REGISTRY_USER --secret type=env,id=GITHUB_REGISTRY_TOKEN,env=GITHUB_REGISTRY_TOKEN --provenance false + --output push-by-digest=true,type=image,push=true + --metadata-file metadata/${BUILD_IMAGE}/${PLATFORM}/build.json containers/eic 2>&1 | tee build.log ; do @@ -569,7 +539,7 @@ user_spack_environment: - gpu needs: - job: version - - job: eic + - job: eic-amd64 parallel: matrix: - BUILD_IMAGE: eic_dev_ @@ -589,6 +559,42 @@ cuda:torch: - python -c "import torch ; print(torch.cuda.is_available()) ;" allow_failure: true +eic:multi: + <<: *eic_parallel + extends: .build + stage: eic + needs: + - eic + script: + - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do + digest=$(jq -r '."containerimage.digest"' ${build_json}) ; + echo ${digest} ; + amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; + done + - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} + - for export_tag in ${EXPORT_TAG:+ + ${IF_BUILD_DEFAULT+ + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} + } + ${IF_BUILD_DEFAULT- + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} + } + } + ${IF_BUILD_NIGHTLY+ + ${NIGHTLY:+ + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} + } + } ; do + docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${export_tag} ; + docker push ${export_tag} ; + done + .singularity: image: name: ${SINGULARITY_IMAGE} From cfe825485cc159a3657193b5f4c852a323132512 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 09:16:22 -0500 Subject: [PATCH 10/44] fix: apk add jq --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4cac8c7..56d9ae94 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -354,6 +354,7 @@ base:multi: needs: - base script: + - apk add jq - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; echo ${digest} ; From 95313d625b0e6732d5b3ff571f52b2036feb66f6 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 10:57:23 -0500 Subject: [PATCH 11/44] test: disable arm again for now --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56d9ae94..29c46e1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -276,7 +276,7 @@ status:pending: BUILD_IMAGE: debian_stable_base PLATFORM: - linux/amd64 - - linux/arm64/v8 + #- linux/arm64/v8 # - BASE_IMAGE: ubuntu:22.04 # BUILD_IMAGE: ubuntu_base # PLATFORM: linux/amd64,linux/arm64/v8 From 5c2305bcd1bf4e9f92a06f19c98df6d7f6921f29 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 11:00:51 -0500 Subject: [PATCH 12/44] test: disable arm again for now (eic) --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29c46e1b..d2664d84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -357,7 +357,6 @@ base:multi: - apk add jq - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - echo ${digest} ; amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; done - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends} @@ -395,7 +394,7 @@ base:multi: RUNTIME_IMAGE: debian_stable_base PLATFORM: - linux/amd64 - - linux/arm64/v8 + #- linux/arm64/v8 # - BUILD_IMAGE: oneapi_ # ENV: # - prod From 8821e1d004060578b18590959a4f2be843aa672f Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 11:12:25 -0500 Subject: [PATCH 13/44] fix: allow building eic container from base by digest --- .gitlab-ci.yml | 14 ++++++++++---- containers/eic/Dockerfile | 7 ++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2664d84..b47646a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -438,13 +438,15 @@ eic: script: - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY= ; fi - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT= ; fi - - apk add envsubst git + - apk add envsubst git jq - source spack.sh ; source spack-packages.sh ; source key4hep-spack.sh ; source eic-spack.sh ; export SPACKPACKAGES_VERSION ; cat mirrors.yaml.in | envsubst > mirrors.yaml + - builder_digest=$(jq -r '."containerimage.digest"' metadata/${BUILDER_IMAGE}/${PLATFORM}/build.json) + - runtime_digest=$(jq -r '."containerimage.digest"' metadata/${RUNTIME_IMAGE}/${PLATFORM}/build.json) - attempts=0 - nocache="" - set -o xtrace ; @@ -467,8 +469,9 @@ eic: --build-arg CAMPAIGNS_SLURM_VERSION=$(sh .ci/resolve_git_ref eic/job_submission_slurm main) --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/ --build-arg BUILDER_IMAGE=${BUILDER_IMAGE} + --build-arg BUILDER_TAG=@${builder_digest} --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE} - --build-arg INTERNAL_TAG=${INTERNAL_TAG} + --build-arg RUNTIME_TAG=@${RUNTIME_digest} --build-arg EIC_CONTAINER_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD) --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA} ${IF_BUILD_DEFAULT+ @@ -568,7 +571,6 @@ eic:multi: script: - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - echo ${digest} ; amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; done - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} @@ -610,14 +612,17 @@ eic:multi: paths: - build/${BUILD_IMAGE}.sif script: + - apk add jq - mkdir build - - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE} + - digest=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${PLATFORM}/build.json) + - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}@${digest} eic_xl:singularity:default: extends: .singularity variables: BUILD_TYPE: default BUILD_IMAGE: eic_xl + PLATFORM: linux/amd64 needs: - job: version - job: eic @@ -638,6 +643,7 @@ eic_xl:singularity:nightly: variables: BUILD_TYPE: nightly BUILD_IMAGE: eic_xl + PLATFORM: linux/amd64 needs: - job: version - job: eic diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index 4e0c5422..f82bc6e2 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -2,8 +2,9 @@ #check=error=true ARG DOCKER_REGISTRY="eicweb/" ARG BUILDER_IMAGE="debian_stable_base" +ARG BUILDER_TAG=":master" ARG RUNTIME_IMAGE="debian_stable_base" -ARG INTERNAL_TAG="master" +ARG RUNTIME_TAG=":master" ## ## This docker build follows two tracks, in order to ensure that we build all packages @@ -34,7 +35,7 @@ ARG INTERNAL_TAG="master" ## builder_concretization_default ## - builder base with concretization of default versions ## ======================================================================================== -FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}:${INTERNAL_TAG} AS builder_concretization_default +FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}${BUILDER_TAG} AS builder_concretization_default ARG TARGETPLATFORM # Open Container Initiative labels @@ -107,7 +108,7 @@ EOF ## runtime_concretization_default ## - runtime base with concretization of default versions (taken from equivalent builder) ## ======================================================================================== -FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}:${INTERNAL_TAG} AS runtime_concretization_default +FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}${RUNTIME_TAG} AS runtime_concretization_default ARG TARGETPLATFORM # Open Container Initiative labels From 2484a1519285950ccb90edce370b116afdb55778 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 11:34:06 -0500 Subject: [PATCH 14/44] fix: use lowercase for local vars --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b47646a3..557f0598 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -471,7 +471,7 @@ eic: --build-arg BUILDER_IMAGE=${BUILDER_IMAGE} --build-arg BUILDER_TAG=@${builder_digest} --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE} - --build-arg RUNTIME_TAG=@${RUNTIME_digest} + --build-arg RUNTIME_TAG=@${runtime_digest} --build-arg EIC_CONTAINER_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD) --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA} ${IF_BUILD_DEFAULT+ From 2900844b56eed6d06ffeeda23de48469a89d2931 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 11:36:36 -0500 Subject: [PATCH 15/44] fix: multi -> multiarch; in separate stage --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 557f0598..c5a141e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -347,10 +347,10 @@ base: let attempts=$attempts+1 ; done -base:multi: +base:multiarch: <<: *base_parallel extends: .build - stage: base + stage: base:multiarch needs: - base script: @@ -562,10 +562,10 @@ cuda:torch: - python -c "import torch ; print(torch.cuda.is_available()) ;" allow_failure: true -eic:multi: +eic:multiarch: <<: *eic_parallel extends: .build - stage: eic + stage: eic:multiarch needs: - eic script: From b1f3f7b156f8f0ffdd87933479248ceff663ef6c Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 11:39:04 -0500 Subject: [PATCH 16/44] fix: add multiarch stages --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c5a141e9..61b0ef14 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,8 +53,10 @@ variables: stages: - status-pending - config - - base ## base OS image - - eic ## EIC container images + - base ## base OS image (build) + - base:multiarch ## base OS image (merge into multiarch) + - eic ## EIC container images (build) + - eic:multiarch ## EIC container images (merge into multiarch) - deploy ## build/deploy singularity images - benchmarks - test From ec01fa10ce51f2690de2e89a896c173ece7a7ef8 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 12:08:37 -0500 Subject: [PATCH 17/44] fix: push internal tag --- .gitlab-ci.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61b0ef14..906cc375 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -249,6 +249,18 @@ status:pending: - metadata/ when: always +## rules for merging the various multiarch digests into a single tag +.multiarch: + extends: .build + script: + - apk add jq + - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do + digest=$(jq -r '."containerimage.digest"' ${build_json}) ; + amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; + done + - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends} + - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} + ## rules for nightly jobs .nightly: rules: @@ -357,11 +369,12 @@ base:multiarch: - base script: - apk add jq - - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do + - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; done - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends} + - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} - for export_tag in ${EXPORT_TAG:+ ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} @@ -571,11 +584,12 @@ eic:multiarch: needs: - eic script: - - for build_json in `find metadata/${BUILD_IMAGE} -name build.json` ; do + - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; + amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}@${digest}" ; done - - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} + - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} + - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} - for export_tag in ${EXPORT_TAG:+ ${IF_BUILD_DEFAULT+ ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} @@ -595,7 +609,7 @@ eic:multiarch: ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} } } ; do - docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${export_tag} ; + docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} ${export_tag} ; docker push ${export_tag} ; done From 7ead42dfe45230cb0438b110a07b1cc8d237625d Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 12:17:27 -0500 Subject: [PATCH 18/44] feat: split amd64 and arm64 matrices --- .gitlab-ci.yml | 78 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 906cc375..c6b3e2fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -280,20 +280,19 @@ status:pending: ## TODO ## oneapi_runtime + eic_xl --> oneapi_prod -.base: &base_parallel +.base-amd64: &base_amd64_parallel parallel: matrix: # - BASE_IMAGE: debian:testing-slim # BUILD_IMAGE: debian_testing_base -# PLATFORM: linux/amd64,linux/arm64/v8 +# PLATFORM: linux/amd64 - BASE_IMAGE: debian:trixie-slim BUILD_IMAGE: debian_stable_base PLATFORM: - linux/amd64 - #- linux/arm64/v8 # - BASE_IMAGE: ubuntu:22.04 # BUILD_IMAGE: ubuntu_base -# PLATFORM: linux/amd64,linux/arm64/v8 +# PLATFORM: linux/amd64 # - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04 # BUILD_IMAGE: oneapi_base # PLATFORM: linux/amd64 @@ -304,8 +303,15 @@ status:pending: BUILD_IMAGE: cuda_runtime PLATFORM: linux/amd64 -base: - <<: *base_parallel +.base-arm64: &base_arm64_parallel + parallel: + matrix: + - BASE_IMAGE: debian:trixie-slim + BUILD_IMAGE: debian_stable_base + PLATFORM: + - linux/arm64/v8 + +.base: extends: .build stage: base needs: @@ -361,12 +367,22 @@ base: let attempts=$attempts+1 ; done + +base-amd64: + <<: *base_amd64_parallel + extends: .base + +base-arm64: + <<: *base_arm64_parallel + extends: .base + base:multiarch: <<: *base_parallel extends: .build stage: base:multiarch needs: - - base + - base-amd64 + - base-arm64 script: - apk add jq - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do @@ -384,7 +400,7 @@ base:multiarch: docker push ${export_tag} ; done -.eic: &eic_parallel +.eic-amd64: &eic_amd64_parallel parallel: matrix: - BUILD_IMAGE: eic_ @@ -409,7 +425,6 @@ base:multiarch: RUNTIME_IMAGE: debian_stable_base PLATFORM: - linux/amd64 - #- linux/arm64/v8 # - BUILD_IMAGE: oneapi_ # ENV: # - prod @@ -443,13 +458,23 @@ base:multiarch: when: manual - when: always -eic: - <<: *eic_parallel +.eic-arm64: &eic_arm64_parallel + parallel: + matrix: + - BUILD_IMAGE: eic_ + ENV: + - ci + BUILD_TYPE: + - default + - nightly + BUILDER_IMAGE: debian_stable_base + RUNTIME_IMAGE: debian_stable_base + PLATFORM: + - linux/arm64/v8 + +.eic: extends: .build stage: eic - needs: - - version - - base script: - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY= ; fi - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT= ; fi @@ -577,12 +602,27 @@ cuda:torch: - python -c "import torch ; print(torch.cuda.is_available()) ;" allow_failure: true +eic-amd64: + <<: *eic_amd64_parallel + extends: .eic + needs: + - version + - base-amd64 + +eic-arm64: + <<: *eic_arm64_parallel + extends: .eic + needs: + - version + - base-arm64 + eic:multiarch: <<: *eic_parallel extends: .build stage: eic:multiarch needs: - - eic + - eic-amd64 + - eic-arm64 script: - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; @@ -641,7 +681,7 @@ eic_xl:singularity:default: PLATFORM: linux/amd64 needs: - job: version - - job: eic + - job: eic-amd64 parallel: matrix: - BUILD_IMAGE: eic_ @@ -662,7 +702,7 @@ eic_xl:singularity:nightly: PLATFORM: linux/amd64 needs: - job: version - - job: eic + - job: eic-amd64 parallel: matrix: - BUILD_IMAGE: eic_ @@ -691,7 +731,7 @@ eic_xl:singularity:nightly: extends: .benchmarks needs: - job: version - - job: eic + - job: eic-amd64 parallel: matrix: - BUILD_IMAGE: eic_ @@ -707,7 +747,7 @@ eic_xl:singularity:nightly: extends: .benchmarks needs: - job: version - - job: eic + - job: eic-amd64 parallel: matrix: - BUILD_IMAGE: eic_ From 9c846fa4f44dcfdbc589e5ebe2d8836a2c6940b6 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 12:37:56 -0500 Subject: [PATCH 19/44] fix: use amd64 anchor in multiarch --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6b3e2fc..a32c1ac5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -377,7 +377,7 @@ base-arm64: extends: .base base:multiarch: - <<: *base_parallel + <<: *base_amd64_parallel extends: .build stage: base:multiarch needs: @@ -617,7 +617,7 @@ eic-arm64: - base-arm64 eic:multiarch: - <<: *eic_parallel + <<: *eic_amd64_parallel extends: .build stage: eic:multiarch needs: From 29202bc0e5112f9d5d88658fbd99eff69dff2a5e Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 13:21:54 -0500 Subject: [PATCH 20/44] fix: eic: mkdir metadata dir --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a32c1ac5..ee4aea93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -485,6 +485,7 @@ base:multiarch: source eic-spack.sh ; export SPACKPACKAGES_VERSION ; cat mirrors.yaml.in | envsubst > mirrors.yaml + - mkdir -p metadata/${BUILD_IMAGE}${ENV}/${PLATFORM} - builder_digest=$(jq -r '."containerimage.digest"' metadata/${BUILDER_IMAGE}/${PLATFORM}/build.json) - runtime_digest=$(jq -r '."containerimage.digest"' metadata/${RUNTIME_IMAGE}/${PLATFORM}/build.json) - attempts=0 @@ -536,7 +537,7 @@ base:multiarch: --secret type=env,id=GITHUB_REGISTRY_TOKEN,env=GITHUB_REGISTRY_TOKEN --provenance false --output push-by-digest=true,type=image,push=true - --metadata-file metadata/${BUILD_IMAGE}/${PLATFORM}/build.json + --metadata-file metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json containers/eic 2>&1 | tee build.log ; do From 769debabc9372d9dfcb0429dc296bdcdec123d3f Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 13:35:37 -0500 Subject: [PATCH 21/44] fix: build exports BUILD_DIGEST; get artifacts in benchmarks trigger --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee4aea93..a6b78cb1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -247,6 +247,8 @@ status:pending: paths: - build.log - metadata/ + reports: + dotenv: build.env when: always ## rules for merging the various multiarch digests into a single tag @@ -366,7 +368,7 @@ status:pending: fi ; let attempts=$attempts+1 ; done - + - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${PLATFORM}/build.json)" | tee -a build.env base-amd64: <<: *base_amd64_parallel @@ -552,6 +554,7 @@ base:multiarch: fi ; let attempts=$attempts+1 ; done + - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json)" | tee -a build.env user_spack_environment: stage: benchmarks @@ -733,6 +736,7 @@ eic_xl:singularity:nightly: needs: - job: version - job: eic-amd64 + artifacts: true parallel: matrix: - BUILD_IMAGE: eic_ @@ -749,6 +753,7 @@ eic_xl:singularity:nightly: needs: - job: version - job: eic-amd64 + artifacts: true parallel: matrix: - BUILD_IMAGE: eic_ From 8a572fee4f7a228097a1f7e7e6a7e31a4bc87fb9 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 13:45:59 -0500 Subject: [PATCH 22/44] fix: use BUILD_DIGEST in triggered benchmarks --- .gitlab-ci.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6b78cb1..73ac8ef9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -771,7 +771,8 @@ benchmarks:geoviewer:default: - if: '$CI_PIPELINE_SOURCE != "schedule" && $EPIC_VERSION != ""' variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-default" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/geoviewer strategy: depend @@ -782,7 +783,8 @@ benchmarks:detector:default: - if: '$CI_PIPELINE_SOURCE != "schedule"' variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-default" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/detector_benchmarks strategy: depend @@ -794,7 +796,8 @@ benchmarks:detector:nightly: - !reference ['.nightly', rules] variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/detector_benchmarks strategy: depend @@ -805,7 +808,8 @@ benchmarks:phyiscs:default: - if: '$CI_PIPELINE_SOURCE != "schedule"' variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-default" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/physics_benchmarks strategy: depend @@ -817,7 +821,8 @@ benchmarks:physics:nightly: - !reference ['.nightly', rules] variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/physics_benchmarks strategy: depend @@ -828,7 +833,8 @@ benchmarks:reconstruction:default: - if: '$CI_PIPELINE_SOURCE != "schedule" && $EICRECON_VERSION == ""' variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-default" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/reconstruction_benchmarks strategy: depend @@ -840,7 +846,8 @@ benchmarks:reconstruction:nightly: - !reference ['.nightly', rules] variables: BENCHMARKS_CONTAINER: eic_ci - BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly" + BENCHMARKS_SIGIL: "@" + BENCHMARKS_TAG: "${BUILD_DIGEST}" trigger: project: EIC/benchmarks/reconstruction_benchmarks strategy: depend From 49bd9775ce038cc62fe2043138cbf9c4065cec3b Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 13:46:37 -0500 Subject: [PATCH 23/44] fix: eic-arm64: allow_failure: true --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73ac8ef9..cd82a939 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -619,6 +619,7 @@ eic-arm64: needs: - version - base-arm64 + allow_failure: true eic:multiarch: <<: *eic_amd64_parallel From df9756dbdd0096c9cf4207385c5455e02a9c7e63 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 14:22:03 -0500 Subject: [PATCH 24/44] fix: base: apk add jq --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd82a939..21ea2001 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -319,6 +319,7 @@ status:pending: needs: - version script: + - apk add jq - source spack.sh ; source spack-packages.sh ; source key4hep-spack.sh ; From 3e2e575ae622fa6087ea1a0700097a9fc1b3999f Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 16:14:25 -0500 Subject: [PATCH 25/44] fix: remove matrix from multiarch jobs --- .gitlab-ci.yml | 87 ++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21ea2001..70bc104a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -380,7 +380,6 @@ base-arm64: extends: .base base:multiarch: - <<: *base_amd64_parallel extends: .build stage: base:multiarch needs: @@ -388,19 +387,21 @@ base:multiarch: - base-arm64 script: - apk add jq - - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do - digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; - done - - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends} - - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} - - for export_tag in ${EXPORT_TAG:+ - ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} - ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - } ; do - docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${export_tag} ; - docker push ${export_tag} ; + - for build_image in $(find metadata -type d) ; do + for build_json in $(find metadata/${build_image} -name build.json) ; do + digest=$(jq -r '."containerimage.digest"' ${build_json}) ; + amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; + done ; + docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ${amends} ; + docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; + for export_tag in ${EXPORT_TAG:+ + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${EXPORT_TAG}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} + } ; do + docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ${export_tag} ; + docker push ${export_tag} ; + done done .eic-amd64: &eic_amd64_parallel @@ -556,6 +557,7 @@ base:multiarch: let attempts=$attempts+1 ; done - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json)" | tee -a build.env + - echo "BUILD_TYPE=${BUILD_TYPE}" | tee -a build.env user_spack_environment: stage: benchmarks @@ -623,41 +625,42 @@ eic-arm64: allow_failure: true eic:multiarch: - <<: *eic_amd64_parallel extends: .build stage: eic:multiarch needs: - eic-amd64 - eic-arm64 script: - - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do - digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}@${digest}" ; - done - - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} - - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} - - for export_tag in ${EXPORT_TAG:+ - ${IF_BUILD_DEFAULT+ - ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - } - ${IF_BUILD_DEFAULT- - ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - } - } - ${IF_BUILD_NIGHTLY+ - ${NIGHTLY:+ - ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} + - for build_image in $(find metadata/ -type d) ; do + for build_json in $(find metadata/${build_image} -name build.json) ; do + digest=$(jq -r '."containerimage.digest"' ${build_json}) ; + amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; + done ; + docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} ; + docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; + for export_tag in ${EXPORT_TAG:+ + ${IF_BUILD_DEFAULT+ + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${EXPORT_TAG}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} + } + ${IF_BUILD_DEFAULT- + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${EXPORT_TAG}-${BUILD_TYPE}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}-${BUILD_TYPE}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}-${BUILD_TYPE}} + } } - } ; do - docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} ${export_tag} ; - docker push ${export_tag} ; - done + ${IF_BUILD_NIGHTLY+ + ${NIGHTLY:+ + ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${NIGHTLY_TAG}} + ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${build_image}:${NIGHTLY_TAG}} + ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${NIGHTLY_TAG}} + } + } ; do + docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ${export_tag} ; + docker push ${export_tag} ; + done ; + done .singularity: image: From c30aac3bd633b3fa90ab8c59a3397e25dd33405a Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 17:26:15 -0500 Subject: [PATCH 26/44] fix: always run multiarch; skip arm64 on trigger --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70bc104a..2210d9c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -378,6 +378,10 @@ base-amd64: base-arm64: <<: *base_arm64_parallel extends: .base + rules: + - if: '$CI_PIPELINE_SOURCE != "trigger"' + when: on_success + - when: never base:multiarch: extends: .build @@ -385,6 +389,8 @@ base:multiarch: needs: - base-amd64 - base-arm64 + rules: + - when: always script: - apk add jq - for build_image in $(find metadata -type d) ; do @@ -622,6 +628,10 @@ eic-arm64: needs: - version - base-arm64 + rules: + - if: '$CI_PIPELINE_SOURCE != "trigger"' + when: on_success + - when: never allow_failure: true eic:multiarch: @@ -630,6 +640,8 @@ eic:multiarch: needs: - eic-amd64 - eic-arm64 + rules: + - when: always script: - for build_image in $(find metadata/ -type d) ; do for build_json in $(find metadata/${build_image} -name build.json) ; do From 6b7c9362c264657b5ba46fcf732b1ff4649b45ff Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 16 Aug 2025 20:46:05 -0500 Subject: [PATCH 27/44] fix: avoid metadata/metadata --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2210d9c5..44d92222 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -393,7 +393,7 @@ base:multiarch: - when: always script: - apk add jq - - for build_image in $(find metadata -type d) ; do + - for build_image in $(ls metadata) ; do for build_json in $(find metadata/${build_image} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; From 612d42fcd6dd23164540d8ebcd4bea3487e14b0d Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 10:12:20 -0500 Subject: [PATCH 28/44] fix: avoid metadata/metadata in eic:multiarch --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 44d92222..2ad89d05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -643,7 +643,7 @@ eic:multiarch: rules: - when: always script: - - for build_image in $(find metadata/ -type d) ; do + - for build_image in $(ls metadata) ; do for build_json in $(find metadata/${build_image} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; From 3caeb971248c3e4b27b6367922ab8b2424e730ee Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 10:30:16 -0500 Subject: [PATCH 29/44] fix: *:multiarch: arm64: optional: true --- .gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ad89d05..c2732560 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -389,8 +389,7 @@ base:multiarch: needs: - base-amd64 - base-arm64 - rules: - - when: always + optional: true script: - apk add jq - for build_image in $(ls metadata) ; do @@ -640,8 +639,7 @@ eic:multiarch: needs: - eic-amd64 - eic-arm64 - rules: - - when: always + optional: true script: - for build_image in $(ls metadata) ; do for build_json in $(find metadata/${build_image} -name build.json) ; do From 2cf7e4be6f1842314ed172fd37f5512b084f7f34 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 10:31:41 -0500 Subject: [PATCH 30/44] fix: *:multiarch: needs: use job hash instead of array --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2732560..725bb447 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -387,8 +387,8 @@ base:multiarch: extends: .build stage: base:multiarch needs: - - base-amd64 - - base-arm64 + - job: base-amd64 + - job: base-arm64 optional: true script: - apk add jq @@ -637,8 +637,8 @@ eic:multiarch: extends: .build stage: eic:multiarch needs: - - eic-amd64 - - eic-arm64 + - job: eic-amd64 + - job: eic-arm64 optional: true script: - for build_image in $(ls metadata) ; do From 2910322ca695c8c4b3470a1edc753d1f663d7331 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 12:20:09 -0500 Subject: [PATCH 31/44] fix: eic:multiarch: apk add jq [ci skip] --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 725bb447..333aa008 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -641,6 +641,7 @@ eic:multiarch: - job: eic-arm64 optional: true script: + - apk add jq - for build_image in $(ls metadata) ; do for build_json in $(find metadata/${build_image} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; From ea4b5f7df2eaba8ada40ed262e65dab93ff703c7 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 12:34:19 -0500 Subject: [PATCH 32/44] fix: docker image history and manifest inspect [ci skip] --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 333aa008..116147d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -370,6 +370,7 @@ status:pending: let attempts=$attempts+1 ; done - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${PLATFORM}/build.json)" | tee -a build.env + - docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${BUILD_DIGEST} base-amd64: <<: *base_amd64_parallel @@ -398,6 +399,7 @@ base:multiarch: amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; done ; docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ${amends} ; + docker manifest inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; for export_tag in ${EXPORT_TAG:+ ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${EXPORT_TAG}} @@ -563,6 +565,7 @@ base:multiarch: done - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json)" | tee -a build.env - echo "BUILD_TYPE=${BUILD_TYPE}" | tee -a build.env + - docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}@${BUILD_DIGEST} user_spack_environment: stage: benchmarks @@ -648,6 +651,7 @@ eic:multiarch: amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; done ; docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} ; + docker manifest inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; for export_tag in ${EXPORT_TAG:+ ${IF_BUILD_DEFAULT+ From c22c031bed2b44a41eac54133a50a06de30e9c6d Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 14:17:11 -0500 Subject: [PATCH 33/44] fix: use BUILDER_SIGIL in Dockerfile too [ci skip] --- .gitlab-ci.yml | 6 ++++-- containers/eic/Dockerfile | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 116147d6..fbbb71cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -521,9 +521,11 @@ base:multiarch: --build-arg CAMPAIGNS_SLURM_VERSION=$(sh .ci/resolve_git_ref eic/job_submission_slurm main) --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/ --build-arg BUILDER_IMAGE=${BUILDER_IMAGE} - --build-arg BUILDER_TAG=@${builder_digest} + --build-arg BUILDER_SIGIL=@ + --build-arg BUILDER_TAG=${builder_digest} --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE} - --build-arg RUNTIME_TAG=@${runtime_digest} + --build-arg RUNTIME_SIGIL=@ + --build-arg RUNTIME_TAG=${runtime_digest} --build-arg EIC_CONTAINER_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD) --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA} ${IF_BUILD_DEFAULT+ diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index f82bc6e2..f77dd315 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -2,9 +2,11 @@ #check=error=true ARG DOCKER_REGISTRY="eicweb/" ARG BUILDER_IMAGE="debian_stable_base" -ARG BUILDER_TAG=":master" +ARG BUILDER_SIGIL=":" +ARG BUILDER_TAG="master" ARG RUNTIME_IMAGE="debian_stable_base" -ARG RUNTIME_TAG=":master" +ARG RUNTIME_SIGIL=":" +ARG RUNTIME_TAG="master" ## ## This docker build follows two tracks, in order to ensure that we build all packages @@ -35,7 +37,7 @@ ARG RUNTIME_TAG=":master" ## builder_concretization_default ## - builder base with concretization of default versions ## ======================================================================================== -FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}${BUILDER_TAG} AS builder_concretization_default +FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}${BUILDER_SIGIL}${BUILDER_TAG} AS builder_concretization_default ARG TARGETPLATFORM # Open Container Initiative labels @@ -108,7 +110,7 @@ EOF ## runtime_concretization_default ## - runtime base with concretization of default versions (taken from equivalent builder) ## ======================================================================================== -FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}${RUNTIME_TAG} AS runtime_concretization_default +FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}${RUNTIME_SIGIL}${RUNTIME_TAG} AS runtime_concretization_default ARG TARGETPLATFORM # Open Container Initiative labels From 5506bbcefec78d1e75dbd71cb32f64f4b0bd1968 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 14:18:15 -0500 Subject: [PATCH 34/44] fix: define BINFMT_IMAGE for multiarch [ci skip] --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fbbb71cc..e8795c17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ variables: ## External images DOCKER_IMAGE: docker.io/docker:28.3.3 SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.5 + BINFMT_IMAGE: multiarch/qemu-user-static ## CUDA version and container operating system CUDA_VERSION: 12.5.1 @@ -231,7 +232,7 @@ status:pending: - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; for arch in aarch64 ; do if ! grep -q enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then - docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes ; + docker run --rm --privileged ${BINFMT_IMAGE} --reset --persistent yes ; fi ; done - docker context create context From 59a27b78427f466fc27334e6797cf9f8aa3f3e00 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 14:30:17 -0500 Subject: [PATCH 35/44] test: use tonistiigi/binfmt:qemu-v9.2.2 for multiarch --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8795c17..36e5fe9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ variables: ## External images DOCKER_IMAGE: docker.io/docker:28.3.3 SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.5 - BINFMT_IMAGE: multiarch/qemu-user-static + BINFMT_IMAGE: tonistiigi/binfmt:qemu-v9.2.2 ## CUDA version and container operating system CUDA_VERSION: 12.5.1 @@ -232,7 +232,7 @@ status:pending: - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; for arch in aarch64 ; do if ! grep -q enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then - docker run --rm --privileged ${BINFMT_IMAGE} --reset --persistent yes ; + docker run --rm --privileged ${BINFMT_IMAGE} --install $arch ; fi ; done - docker context create context From c0f8fd2e194d947799c941d8c25185bdc8bc8c73 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 14:52:14 -0500 Subject: [PATCH 36/44] fix: run BINFMT_IMAGE --version --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36e5fe9e..35d0e7bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -235,6 +235,7 @@ status:pending: docker run --rm --privileged ${BINFMT_IMAGE} --install $arch ; fi ; done + - docker run --rm --privileged ${BINFMT_IMAGE} --version - docker context create context # The docker-container cache persistence applies to builders of the same name, # so do not change the name to something that has the pipeline or job id. From cc2c5f2a47cba8f48a6aaeffc404497c5c8108be Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 15:09:03 -0500 Subject: [PATCH 37/44] fix: ensure BUILD_DIGEST is defined --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35d0e7bf..1d5105e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -371,8 +371,9 @@ status:pending: fi ; let attempts=$attempts+1 ; done - - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${PLATFORM}/build.json)" | tee -a build.env + - BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${PLATFORM}/build.json) - docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${BUILD_DIGEST} + - echo "BUILD_DIGEST=${BUILD_DIGEST}" | tee -a build.env base-amd64: <<: *base_amd64_parallel @@ -567,7 +568,8 @@ base:multiarch: fi ; let attempts=$attempts+1 ; done - - echo "BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json)" | tee -a build.env + - BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json) + - echo "BUILD_DIGEST=${BUILD_DIGEST}" | tee -a build.env - echo "BUILD_TYPE=${BUILD_TYPE}" | tee -a build.env - docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}@${BUILD_DIGEST} From ff4bfee4bc455085d5d4818c8e25c65195a57f04 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 15:32:36 -0500 Subject: [PATCH 38/44] fix: disable docker image history for now --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d5105e0..b120f618 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -372,7 +372,7 @@ status:pending: let attempts=$attempts+1 ; done - BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}/${PLATFORM}/build.json) - - docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${BUILD_DIGEST} + #- docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${BUILD_DIGEST} - echo "BUILD_DIGEST=${BUILD_DIGEST}" | tee -a build.env base-amd64: @@ -569,9 +569,9 @@ base:multiarch: let attempts=$attempts+1 ; done - BUILD_DIGEST=$(jq -r '."containerimage.digest"' metadata/${BUILD_IMAGE}${ENV}/${PLATFORM}/build.json) + #- docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}@${BUILD_DIGEST} - echo "BUILD_DIGEST=${BUILD_DIGEST}" | tee -a build.env - echo "BUILD_TYPE=${BUILD_TYPE}" | tee -a build.env - - docker image history ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}@${BUILD_DIGEST} user_spack_environment: stage: benchmarks From c2dfb23cf6a33e0609d03c1dd9e090d1d0c2b1e7 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 17 Aug 2025 17:22:31 -0500 Subject: [PATCH 39/44] fix: *:multiarch: needs: job: version --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b120f618..dfb71d63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -391,6 +391,7 @@ base:multiarch: extends: .build stage: base:multiarch needs: + - job: version - job: base-amd64 - job: base-arm64 optional: true @@ -646,6 +647,7 @@ eic:multiarch: extends: .build stage: eic:multiarch needs: + - job: version - job: eic-amd64 - job: eic-arm64 optional: true From d94d8a08ea343b959d65e92d19f7865fa06fbc8b Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Mon, 18 Aug 2025 16:04:42 -0500 Subject: [PATCH 40/44] fix: docker buildx imagetools create for tagging manifests --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dfb71d63..714c375e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -410,7 +410,8 @@ base:multiarch: ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} } ; do - docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ${export_tag} ; + docker buildx imagetools create --tag ${export_tag} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; + docker buildx imagetools inspect ${export_tag} docker push ${export_tag} ; done done @@ -680,7 +681,8 @@ eic:multiarch: ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${NIGHTLY_TAG}} } } ; do - docker tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ${export_tag} ; + docker buildx imagetools create --tag ${export_tag} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; + docker buildx imagetools inspect ${export_tag} docker push ${export_tag} ; done ; done From e360700a7a78db042ef8ad39836a80e7e4f35f6a Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Tue, 2 Sep 2025 18:34:20 -0500 Subject: [PATCH 41/44] test: comment out docker buildx imagetools inspect --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 714c375e..1bfa359a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -411,7 +411,7 @@ base:multiarch: ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} } ; do docker buildx imagetools create --tag ${export_tag} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; - docker buildx imagetools inspect ${export_tag} + #docker buildx imagetools inspect ${export_tag} docker push ${export_tag} ; done done From 90fbbc35497698418f35730ad0aaa8cf213ee653 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Tue, 2 Sep 2025 22:59:30 -0500 Subject: [PATCH 42/44] fix: re-enable docker buildx imagetools inspect with semicolon --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bfa359a..53f50ccb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -411,7 +411,7 @@ base:multiarch: ${GH_PUSH:+${GH_REGISTRY}/${GH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} } ; do docker buildx imagetools create --tag ${export_tag} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; - #docker buildx imagetools inspect ${export_tag} + docker buildx imagetools inspect ${export_tag} ; docker push ${export_tag} ; done done From 36468a88c5ed8a9adb488a45469baa9fe245d2e0 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 13 Nov 2025 17:45:25 -0600 Subject: [PATCH 43/44] Modify multi-arch manifest merging to use docker buildx imagetools --- .gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53f50ccb..c9b6041a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -258,12 +258,13 @@ status:pending: extends: .build script: - apk add jq - - for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do + - sources="" ; + for build_json in $(find metadata/${BUILD_IMAGE} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; + sources="${sources} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}@${digest}" ; done - - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${amends} - - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} + - docker buildx imagetools create --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ${sources} + - docker buildx imagetools inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} ## rules for nightly jobs .nightly: @@ -398,13 +399,13 @@ base:multiarch: script: - apk add jq - for build_image in $(ls metadata) ; do + sources="" ; for build_json in $(find metadata/${build_image} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; + sources="${sources} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; done ; - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ${amends} ; - docker manifest inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; + docker buildx imagetools create --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ${sources} ; + docker buildx imagetools inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; for export_tag in ${EXPORT_TAG:+ ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${EXPORT_TAG}} ${DH_PUSH:+${DH_REGISTRY}/${DH_REGISTRY_USER}/${build_image}:${EXPORT_TAG}} @@ -412,7 +413,6 @@ base:multiarch: } ; do docker buildx imagetools create --tag ${export_tag} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG} ; docker buildx imagetools inspect ${export_tag} ; - docker push ${export_tag} ; done done @@ -655,13 +655,13 @@ eic:multiarch: script: - apk add jq - for build_image in $(ls metadata) ; do + sources="" ; for build_json in $(find metadata/${build_image} -name build.json) ; do digest=$(jq -r '."containerimage.digest"' ${build_json}) ; - amends="${amends} --amend ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; + sources="${sources} ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}@${digest}" ; done ; - docker manifest create ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ${amends} ; - docker manifest inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; - docker manifest push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; + docker buildx imagetools create --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ${sources} ; + docker buildx imagetools inspect ${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${INTERNAL_TAG}-${BUILD_TYPE} ; for export_tag in ${EXPORT_TAG:+ ${IF_BUILD_DEFAULT+ ${CI_PUSH:+${CI_REGISTRY}/${CI_PROJECT_PATH}/${build_image}:${EXPORT_TAG}} From fbecf75daf4faf2cee9d05a9286250918c5936e5 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 13 Nov 2025 17:51:44 -0600 Subject: [PATCH 44/44] Fix name of upstream job for user_spack_environment --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c9b6041a..b76cada6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -579,7 +579,7 @@ user_spack_environment: stage: benchmarks needs: - job: version - - job: eic + - job: eic-amd64 parallel: matrix: - BUILD_IMAGE: eic_