ppc64le(jupyter/trustyai): create multiarch build for ppc64le (#2465)

Signed-off-by: Md. Shafi Hussain <Md.Shafi.Hussain@ibm.com>

Author: Shafi-Hussain

.tekton/odh-workbench-jupyter-trustyai-cpu-py312-ubi9-pull-request.yaml

@@ -36,6 +36,7 @@ spec:
   - name: build-platforms
     value:
     - linux/x86_64
+    - linux/ppc64le
   - name: dockerfile
     value: jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu
   - name: path-context

jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu

@@ -15,6 +15,27 @@ RUN curl -Lo mongodb-cli-mongocli-v${MONGOCLI_VERSION}.zip https://github.com/mo
 RUN unzip ./mongodb-cli-mongocli-v${MONGOCLI_VERSION}.zip
 RUN cd ./mongodb-cli-mongocli-v${MONGOCLI_VERSION}/ && \
     CGO_ENABLED=1 GOOS=linux go build -a -tags strictfipsruntime -o /tmp/mongocli ./cmd/mongocli/
+####################
+# wheel-cache-base #
+####################
+FROM ${BASE_IMAGE} AS whl-cache
+
+USER root
+ENV HOME=/root
+WORKDIR /root
+
+ARG TRUSTYAI_SOURCE_CODE=jupyter/trustyai/ubi9-python-3.12
+
+COPY ${TRUSTYAI_SOURCE_CODE}/pylock.toml .
+COPY ${TRUSTYAI_SOURCE_CODE}/devel_env_setup.sh .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    pip install --no-cache uv && \
+    # the devel script is ppc64le specific - sets up build-time dependencies
+    source ./devel_env_setup.sh && \
+    # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
+    #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
+    UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml
 
 ####################
 # cpu-base         #
@@ -117,6 +138,8 @@ FROM jupyter-datascience AS jupyter-trustyai
 ARG DATASCIENCE_SOURCE_CODE=jupyter/datascience/ubi9-python-3.12
 ARG TRUSTYAI_SOURCE_CODE=jupyter/trustyai/ubi9-python-3.12
 
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
+
 LABEL name="odh-notebook-jupyter-trustyai-ubi9-python-3.12" \
     summary="Jupyter trustyai notebook image for ODH notebooks" \
     description="Jupyter trustyai notebook image with base Python 3.12 builder image based on UBI9 for ODH notebooks" \
@@ -134,16 +157,39 @@ RUN INSTALL_PKGS="java-17-openjdk" && \
     dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
     dnf -y clean all --enablerepo='*'
 
-USER 1001
-
 # Install Python packages and Jupyterlab extensions from requirements.txt
 COPY ${TRUSTYAI_SOURCE_CODE}/pylock.toml ./
 
-RUN echo "Installing softwares and packages" && \
-    # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
-    #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
-    uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml && \
-    # setup path for runtime configuration
+# install openblas for ppc64le
+RUN --mount=type=cache,from=whl-cache,source=/root/OpenBLAS/,target=/OpenBlas/,rw \
+    bash -c ' \
+        if [[ $(uname -m) == "ppc64le" ]]; then \
+            dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm; \
+            dnf install -y libraqm libimagequant; \
+            PREFIX=/usr/ make install -C /OpenBlas; \
+        fi '
+
+# Install packages and cleanup
+# install packages as USER 0 (this will allow us to consume uv cache)
+RUN --mount=type=cache,from=whl-cache,source=/wheelsdir/,target=/wheelsdir/,rw \
+    --mount=type=cache,target=/root/.cache/uv \
+    bash -c ' \
+        if [[ $(uname -m) == "ppc64le" ]]; then \
+            UV_LINK_MODE=copy uv pip install /wheelsdir/*.whl accelerate --cache-dir /root/.cache/uv; \
+        fi '
+RUN --mount=type=cache,target=/root/.cache/uv \
+    echo "Installing softwares and packages" && \
+    # we can ensure wheels are consumed from the cache only by restricting internet access for uv install with '--offline' flag
+    UV_LINK_MODE=copy uv pip install --cache-dir /root/.cache/uv --requirements=./pylock.toml && \
+    # Note: debugpy wheel availabe on pypi (in uv cache) is none-any but bundles amd64.so files
+    #       Build debugpy from source instead
+    UV_LINK_MODE=copy uv pip install --no-cache git+https://github.com/microsoft/debugpy.git@v$(grep -A1 '\"debugpy\"' ./pylock.toml | grep -Eo '\b[0-9\.]+\b') && \
+    # change ownership to default user (all packages were installed as root and has root:root ownership \
+    chown -R 1001:0 /opt/app-root/
+
+USER 1001
+
+RUN # setup path for runtime configuration \
     mkdir /opt/app-root/runtimes && \
     # Remove default Elyra runtime-images \
     rm /opt/app-root/share/jupyter/metadata/runtime-images/*.json && \

jupyter/trustyai/ubi9-python-3.12/devel_env_setup.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -eoux pipefail
+
+#####################################################################################################
+# This script is expected to be run on ppc64le hosts as `root`                                      #
+# It installs the required build-time dependencies for python wheels                                #
+# OpenBlas is built from source (instead of distro provided) with recommended flags for performance #
+#####################################################################################################
+WHEELS_DIR=/wheelsdir
+mkdir -p ${WHEELS_DIR}
+if [[ $(uname -m) == "ppc64le" ]]; then
+    CURDIR=$(pwd)
+
+    # install development packages
+    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+    dnf install -y fribidi-devel gcc-toolset-13 lcms2-devel libimagequant-devel \
+        libraqm-devel openjpeg2-devel tcl-devel tk-devel unixODBC-devel
+
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+    source /opt/rh/gcc-toolset-13/enable
+    source $HOME/.cargo/env
+    
+    uv pip install cmake
+
+    export MAX_JOBS=${MAX_JOBS:-$(nproc)}
+    export OPENBLAS_VERSION=${OPENBLAS_VERSION:-0.3.30}
+
+    # Install OpenBlas
+    # IMPORTANT: Ensure Openblas is installed in the final image
+    cd /root
+    curl -L https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.tar.gz | tar xz
+    # rename directory for mounting (without knowing version numbers) in multistage builds
+    mv OpenBLAS-${OPENBLAS_VERSION}/ OpenBLAS/
+    cd OpenBLAS/
+    make -j${MAX_JOBS} TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
+    make install
+    cd ..
+
+    # set path for openblas
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/OpenBLAS/lib/:/usr/local/lib64:/usr/local/lib
+    export PKG_CONFIG_PATH=$(find / -type d -name "pkgconfig" 2>/dev/null | tr '\n' ':')
+    export CMAKE_ARGS="-DPython3_EXECUTABLE=python"
+    export CMAKE_POLICY_VERSION_MINIMUM=3.5
+
+    TMP=$(mktemp -d)
+
+    # Torch
+    cd ${CURDIR}
+    TORCH_VERSION=$(grep -A1 '"torch"' pylock.toml | grep -Eo '\b[0-9\.]+\b')
+    cd ${TMP}
+    git clone --recursive https://github.com/pytorch/pytorch.git -b v${TORCH_VERSION}
+    cd pytorch
+    uv pip install -r requirements.txt
+    python setup.py develop
+    rm -f dist/torch*+git*whl
+    MAX_JOBS=${MAX_JOBS:-$(nproc)} \
+        PYTORCH_BUILD_VERSION=${TORCH_VERSION} PYTORCH_BUILD_NUMBER=1 uv build --wheel --out-dir ${WHEELS_DIR}
+
+    cd ${CURDIR}
+    # Pyarrow
+    PYARROW_VERSION=$(grep -A1 '"pyarrow"' pylock.toml | grep -Eo '\b[0-9\.]+\b')
+    cd ${TMP}
+    git clone --recursive https://github.com/apache/arrow.git -b apache-arrow-${PYARROW_VERSION}
+    cd arrow/cpp
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=release \
+        -DCMAKE_INSTALL_PREFIX=/usr/local \
+        -DARROW_PYTHON=ON \
+        -DARROW_BUILD_TESTS=OFF \
+        -DARROW_JEMALLOC=ON \
+        -DARROW_BUILD_STATIC="OFF" \
+        -DARROW_PARQUET=ON \
+        .. && \
+    make install -j ${MAX_JOBS:-$(nproc)} && \
+    cd ../../python/ && \
+    uv pip install -v -r requirements-wheel-build.txt && \
+    PYARROW_PARALLEL=${PYARROW_PARALLEL:-$(nproc)} \
+    python setup.py build_ext \
+    --build-type=release --bundle-arrow-cpp \
+    bdist_wheel --dist-dir ${WHEELS_DIR}
+
+    ls -ltr ${WHEELS_DIR}
+
+    cd ${CURDIR}
+    uv pip install --refresh ${WHEELS_DIR}/*.whl accelerate==$(grep -A1 '"accelerate"' pylock.toml | grep -Eo '\b[0-9\.]+\b')
+
+    uv pip list
+    cd ${CURDIR}
+else
+    # only for mounting on non-ppc64le
+    mkdir -p /root/OpenBLAS/
+fi