Upgrade Spack to v1.1.0

.github/workflows/build-push.yml

@@ -71,6 +71,7 @@ jobs:
           PLATFORM: linux/arm64
           runner: ubuntu-24.04-arm
           arch: arm64
+      fail-fast: false
     steps:
       - name: Checkout 
         uses: actions/checkout@v4
@@ -297,6 +298,7 @@ jobs:
         - arch: arm64
           runner: ubuntu-24.04-arm
           PLATFORM: linux/arm64
+      fail-fast: false
     steps:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@v1.3.1

containers/debian/Dockerfile

@@ -239,6 +239,7 @@ target=${target[${TARGETPLATFORM}]}
 spack config --scope site add "packages:all:require:[target=${target}]"
 spack config --scope site add "packages:all:target:[${target}]"
 spack config blame packages
+mkdir -p $HOME/.spack/  # workaround for Spack not creating config directory automatically in some versions (see https://github.com/spack/spack/issues/51564)
 spack config --scope user add "config:suppress_gpg_warnings:true"
 spack config --scope user add "config:build_jobs:${jobs}"
 spack config --scope user add "config:db_lock_timeout:${jobs}00"

containers/eic/Dockerfile

@@ -46,6 +46,14 @@ ARG ENV=xl
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
 ARG SPACK_FLAGS="--backtrace"
 ARG SPACK_INSTALL_FLAGS="--no-check-signature --show-log-on-error --yes-to-all"
+# The SPACK_DUPLICATE_ALLOWLIST specifies packages that are allowed to have duplicate concretizations.
+# This is necessary for certain packages (e.g., epic, llvm, py-setuptools, py-urllib3, py-dask) due to
+# their complex dependency trees or compatibility requirements. The new additions (py-dask-awkward,
+# py-dask-histogram, py-distributed, py-requests) are included because they may be required in multiple
+# versions or configurations by different packages in the environment, and allowing duplicates prevents
+# build failures or conflicts during concretization. This list should be reviewed periodically to ensure
+# only necessary packages are included.
+ARG SPACK_DUPLICATE_ALLOWLIST="epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests"
 ENV SPACK_COLOR="always"
 ENV GIT_TERMINAL_PROMPT=0
 
@@ -57,7 +65,7 @@ spack external find --not-buildable --scope env:${SPACK_ENV} --path /usr/local/c
 spack external find --scope env:${SPACK_ENV} llvm
 spack concretize --force
 spack --color=never find --long --no-groups --show-concretized --format "{name}" \
-| uniq -D -f2 | grep -v -w -e "\(epic\|llvm\|py-setuptools\|py-urllib3\)" \
+| uniq -D -f2 | grep -Evw "(${SPACK_DUPLICATE_ALLOWLIST})" \
 | tee /tmp/duplicates.txt
 if [ -s /tmp/duplicates.txt ] ; then
   echo "Duplicate packages found"
@@ -98,6 +106,7 @@ export CCACHE_DIR=/ccache
 mkdir -p /var/cache/spack/blobs/sha256/
 find /var/cache/spack/blobs/sha256/ -ignore_readdir_race -atime +7 -delete
 spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS}
+spack clean --downloads --stage
 ccache --show-stats
 ccache --zero-stats
 EOF
@@ -193,7 +202,7 @@ if [ -n "${JUGGLER_SHA}" ] ; then
 fi
 spack concretize --force
 spack --color=never find --long --no-groups --show-concretized --format "{name}" \
-| uniq -D -f2 | grep -v -w -e "\(epic\|llvm\|py-setuptools\|py-urllib3\)" \
+| uniq -D -f2 | grep -Evw "(${SPACK_DUPLICATE_ALLOWLIST})" \
 | tee /tmp/duplicates.txt
 if [ -s /tmp/duplicates.txt ] ; then
   echo "Duplicate packages found"
@@ -232,6 +241,7 @@ RUN --mount=type=cache,target=/ccache,id=ccache-${TARGETPLATFORM}              \
 set -e
 export CCACHE_DIR=/ccache
 spack ${SPACK_FLAGS} install ${SPACK_INSTALL_FLAGS}
+spack clean --downloads --stage
 spack gc --yes-to-all go go-bootstrap rust rust-bootstrap py-setuptools-rust py-maturin
 ccache --show-stats
 ccache --zero-stats

spack-environment/ci/spack.yaml

@@ -2,14 +2,15 @@ spack:
   include:
   - ../concretizer.yaml
   - ../packages.yaml
+  - ../packages_root_without_opengl.yaml
   - ../view.yaml
   specs:
   - acts
   - actsvg
   - afterburner
   - cmake
   - covfie
-  - dd4hep -ddeve
+  - dd4hep
   - edm4hep
   - eigen
   - fastjet
@@ -66,7 +67,7 @@ spack:
   - py-toml
   - py-uproot
   - py-vector
-  - root -opengl -webgui
+  - root
   - snakemake
   - spdlog
   - xrootd

spack-environment/packages.yaml

@@ -251,9 +251,6 @@ packages:
     require:
     - '@6.5.5'
     - +python
-  llvm:
-    require:
-    - ~gold
   madx:
     require:
     - '@5.08.01:'
@@ -330,7 +327,10 @@ packages:
   pyrobird:
     require:
     - '@0.1.23:'
-    - +batch +xrootd
+    - +xrootd
+    - spec: ~batch
+      when: '@:0.2.6'
+      message: "through at least 0.2.6, +batch requires py-pyppeteer which requires py-urllib3@1"
   pythia8:
     require:
     - '@8.315'
@@ -356,6 +356,9 @@ packages:
   py-epic-capybara:
     require:
     - '@git.ef2a8790312b9b697655b28a07b353e601bfa3b8'
+  py-flatbuffers:
+    require:
+    - '@25.9.23:'
   py-hepunits:
     require:
     - '@2.3.2:'

spack-environment/packages_root_with_opengl.yaml

@@ -0,0 +1,4 @@
+packages:
+  root:
+    require:
+    - +opengl +webgui

spack-environment/packages_root_without_opengl.yaml

@@ -0,0 +1,4 @@
+packages:
+  root:
+    require:
+    - ~opengl ~webgui

spack-environment/xl/spack.yaml

@@ -2,9 +2,8 @@ spack:
   include:
   - ../concretizer.yaml
   - ../packages.yaml
+  - ../packages_root_with_opengl.yaml
   - ../view.yaml
-  concretizer:
-    unify: when_possible # multiple epic versions
   specs:
   - acts
   - actsvg
@@ -23,7 +22,7 @@ spack:
   - cppcoro
   - dawn
   - dawncut
-  - dd4hep +ddeve
+  - dd4hep
   - east
   - edm4hep
   - eic-smear
@@ -116,7 +115,7 @@ spack:
   - py-wurlitzer
   - py-yapf
   - rivet
-  - root +opengl +webgui +x
+  - root
   - sherpa
   - simsipm
   - slurm sysconfdir=/etc/slurm

spack-packages.sh

@@ -45,6 +45,7 @@ c4d983b764d7b1ee2b63da79f5f25365ac61ce7a
 44da889cc86bb8a5315c729a7c79f2c002c9c951
 5c37f836753e8f9683fdc547f3661c5045abcbd1
 c75e10845431600b163c597545bd099e427c62f5
+2ba80e697faf80613b038615b2345b7a777cc438
 ---
 ## Optional hash table with comma-separated file list
 read -r -d '' SPACKPACKAGES_CHERRYPICKS_FILES <<- \
@@ -90,4 +91,5 @@ read -r -d '' SPACKPACKAGES_CHERRYPICKS_FILES <<- \
 ## c4d983b764d7b1ee2b63da79f5f25365ac61ce7a: Julia: add v1.11.6
 ## 44da889cc86bb8a5315c729a7c79f2c002c9c951: Julia: add v1.11.7
 ## 5c37f836753e8f9683fdc547f3661c5045abcbd1: root: require openblas ~ilp64 symbol_suffix=none when ^openblas
-## c75e10845431600b163c597545bd099e427c62f5: dd4hep: v1.33
+## c75e10845431600b163c597545bd099e427c62f5: dd4hep: v1.33
+## 2ba80e697faf80613b038615b2345b7a777cc438: py-flatbuffers: add v25.9.23

spack.sh

@@ -3,7 +3,7 @@ SPACK_ORGREPO="spack/spack"
 
 ## Spack github version, e.g. v0.18.1 or commit hash
 ## note: nightly builds will use e.g. releases/v1.0
-SPACK_VERSION="v1.0.2"
+SPACK_VERSION="v1.1.0"
 
 ## Space-separated list of spack cherry-picks
 read -r -d '' SPACK_CHERRYPICKS <<- \