Merge branch 'kubeflow:master' into migrate-images-to-ghcr

Author: mahdikhashan

.github/workflows/main.yml

@@ -32,7 +32,8 @@ jobs:
         run: make test
   e2e:
     name: E2E
-    runs-on: ubuntu-latest
+    runs-on:
+      labels: ubuntu-latest-16-cores
     # Pull requests from the same repository won't trigger this checks as they were already triggered by the push
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:

.github/workflows/mpi-operator-docker-image-publish.yml

@@ -15,7 +15,8 @@ env:
 
 jobs:
   build-push-docker-image:
-    runs-on: ubuntu-latest
+    runs-on:
+      labels: ubuntu-latest-16-cores
     steps:
       - name: Checkout
         uses: actions/checkout@v3

Makefile

@@ -105,12 +105,22 @@ images:
 .PHONY: test_images
 test_images:
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(PLATFORMS) --build-arg port=${BASE_IMAGE_SSH_PORT} -t ${REGISTRY}/base:${RELEASE_VERSION} build/base
+	$(MAKE) -j3 test_images_openmpi test_images_intel test_images_mpich
+
+.PHONY: test_images_openmpi
+test_images_openmpi:
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(PLATFORMS) --build-arg BASE_LABEL=${RELEASE_VERSION} -t ${REGISTRY}/openmpi:${RELEASE_VERSION} build/base -f build/base/openmpi.Dockerfile
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(PLATFORMS) -t ${REGISTRY}/openmpi-builder:${RELEASE_VERSION} build/base -f build/base/openmpi-builder.Dockerfile
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(PLATFORMS) --build-arg BASE_LABEL=${RELEASE_VERSION} -t ${REGISTRY}/mpi-pi:${RELEASE_VERSION}-openmpi examples/v2beta1/pi
+
+.PTHONY: test_images_intel
+test_images_intel:
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(INTEL_PLATFORMS) --build-arg BASE_LABEL=${RELEASE_VERSION} -t ${REGISTRY}/intel:${RELEASE_VERSION} build/base -f build/base/intel.Dockerfile
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(INTEL_PLATFORMS) -t ${REGISTRY}/intel-builder:${RELEASE_VERSION} build/base -f build/base/intel-builder.Dockerfile
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(INTEL_PLATFORMS) --build-arg BASE_LABEL=${RELEASE_VERSION} -t ${REGISTRY}/mpi-pi:${RELEASE_VERSION}-intel examples/v2beta1/pi -f examples/v2beta1/pi/intel.Dockerfile
+
+.PHOTNY: test_images_mpich
+test_images_mpich:
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(MPICH_PLATFORMS) --build-arg BASE_LABEL=${RELEASE_VERSION} -t ${REGISTRY}/mpich:${RELEASE_VERSION} build/base -f build/base/mpich.Dockerfile
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(MPICH_PLATFORMS) -t ${REGISTRY}/mpich-builder:${RELEASE_VERSION} build/base -f build/base/mpich-builder.Dockerfile
 	${IMG_BUILDER} build $(BUILD_ARGS) --platform $(MPICH_PLATFORMS) --build-arg BASE_LABEL=${RELEASE_VERSION} -t ${REGISTRY}/mpi-pi:${RELEASE_VERSION}-mpich examples/v2beta1/pi -f examples/v2beta1/pi/mpich.Dockerfile

build/base/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bookworm
+FROM debian:trixie
 
 ARG port=2222
 

build/base/intel-builder.Dockerfile

@@ -2,23 +2,25 @@ FROM bash AS downloader
 
 RUN wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -O key.PUB
 
-FROM debian:bookworm
+FROM debian:trixie
 
 COPY --from=downloader key.PUB /tmp/key.PUB
 
 # Install Intel oneAPI keys.
 RUN apt update \
-    && apt install -y --no-install-recommends gnupg2 ca-certificates \
-    && apt-key add /tmp/key.PUB \
+    && apt install -y --no-install-recommends gnupg2 ca-certificates apt-transport-https \
+    && gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg /tmp/key.PUB \
     && rm /tmp/key.PUB \
-    && echo "deb https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list \
-    && apt remove -y gnupg2 ca-certificates \
-    && apt autoremove -y \
+    # TODO (tenzen-y): Once Intel OneAPI supports new parsable PGP format for apt, we should remove `trusted=yes` option.
+    # REF: https://github.com/kubeflow/mpi-operator/issues/691
+    && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg trusted=yes] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list \
     && apt update \
     && apt install -y --no-install-recommends \
         libstdc++-12-dev binutils procps clang \
         intel-oneapi-compiler-dpcpp-cpp \
         intel-oneapi-mpi-devel-2021.13 \
+    && apt remove -y gnupg2 ca-certificates apt-transport-https \
+    && apt autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
 ENV I_MPI_CC=clang I_MPI_CXX=clang++

build/base/intel.Dockerfile

@@ -10,16 +10,18 @@ COPY --from=downloader key.PUB /tmp/key.PUB
 
 # Install Intel oneAPI keys.
 RUN apt update \
-    && apt install -y --no-install-recommends gnupg2 ca-certificates \
-    && apt-key add /tmp/key.PUB \
+    && apt install -y --no-install-recommends gnupg2 ca-certificates apt-transport-https \
+    && gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg /tmp/key.PUB \
     && rm /tmp/key.PUB \
-    && echo "deb https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list \
-    && apt remove -y gnupg2 ca-certificates \
-    && apt autoremove -y \
+    # TODO (tenzen-y): Once Intel OneAPI supports new parsable PGP format for apt, we should remove `trusted=yes` option.
+    # REF: https://github.com/kubeflow/mpi-operator/issues/691
+    && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg trusted=yes] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list \
     && apt update \
     && apt install -y --no-install-recommends \
         dnsutils \
         intel-oneapi-mpi-2021.13 \
+    && apt remove -y gnupg2 ca-certificates \
+    && apt autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
 COPY entrypoint.sh /entrypoint.sh

build/base/mpich-builder.Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bookworm as builder
+FROM debian:trixie as builder
 
 RUN apt update \
     && apt install -y --no-install-recommends \

build/base/openmpi-builder.Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bookworm as builder
+FROM debian:trixie as builder
 
 RUN apt update \
     && apt install -y --no-install-recommends \