From 28f0ae1df4b98763c35827e25b74bd256b408071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 08:17:48 +0100 Subject: [PATCH 01/12] RHAIENG-948: fix(ppc): build failure due to missing curl add `libcurl-devel` to build dependencies for `ppc64le` and `s390x` in pyarrow and onnx builds ``` -- Building AWS C++ SDK from source CMake Error at /usr/share/cmake/Modules/FindPackageHandleStandardArgs.cmake:230 (message): Could NOT find CURL (missing: CURL_LIBRARY CURL_INCLUDE_DIR) ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index 6228007343..be6fbc0263 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -144,7 +144,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \ set -Eeuxo pipefail if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then # Install build dependencies (shared for pyarrow and onnx) - dnf install -y cmake make gcc-c++ pybind11-devel wget + packages=( + cmake make gcc-c++ pybind11-devel wget + # pyarrow -DARROW_S3=ON pulls in AWS C++ SDK + # libcurl-devel is required to build AWS C++ SDK + libcurl-devel + ) + dnf install -y "${packages[@]}" dnf clean all # Build and collect pyarrow wheel git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git From ef8a1c00b439417a18492b844f793560d826538b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 08:35:03 +0100 Subject: [PATCH 02/12] RHAIENG-948: fix(ppc): build failure due to missing openssl add `libcurl-devel` to build dependencies for `ppc64le` and `s390x` in pyarrow and onnx builds ``` -- Configuring done (1.8s) -- Generating done (0.1s) CMake Error at cmake_modules/ThirdpartyToolchain.cmake:5282 (set_property): The link interface of target "AWS::aws-c-cal" contains: OpenSSL::Crypto but the target was not found. Possible reasons include: * There is a typo in the target name. * A find_package call is missing for an IMPORTED target. * An ALIAS target is missing. ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index be6fbc0263..183fff4ec5 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -148,7 +148,7 @@ if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then cmake make gcc-c++ pybind11-devel wget # pyarrow -DARROW_S3=ON pulls in AWS C++ SDK # libcurl-devel is required to build AWS C++ SDK - libcurl-devel + libcurl-devel openssl-devel ) dnf install -y "${packages[@]}" dnf clean all From 65c8df91d27d1018e8b1eddd8bdc4f5a93a65997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 08:50:56 +0100 Subject: [PATCH 03/12] RHAIENG-948: fix(ppc): build failure due to missing cython in aipcc wheel index ``` + pip install --no-cache-dir -r requirements-build.txt Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/ Ignoring oldest-supported-numpy: markers 'python_version < "3.9"' don't match your environment ERROR: Could not find a version that satisfies the requirement cython>=0.29.31 (from versions: none) ERROR: No matching distribution found for cython>=0.29.31 ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index 183fff4ec5..e13a23e98f 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -179,7 +179,8 @@ if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then make -j$(nproc) VERBOSE=1 make install -j$(nproc) cd ../../python - pip install --no-cache-dir -r requirements-build.txt + # aipcc index is missing cython, and maybe more + pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements-build.txt PYARROW_WITH_PARQUET=1 \ PYARROW_WITH_DATASET=1 \ PYARROW_WITH_FILESYSTEM=1 \ From a7c3f57a18c77df1a124de7f882cd736dd0f08de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 09:35:40 +0100 Subject: [PATCH 04/12] RHAIENG-948: fix(ppc): build failure due to missing numpy in aipcc wheel index ``` + pip install --no-cache-dir -r requirements.txt Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/ ERROR: Could not find a version that satisfies the requirement numpy>=1.22 (from versions: none) ERROR: No matching distribution found for numpy>=1.22 subprocess exited with status 1 ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index e13a23e98f..a055611966 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -231,7 +231,8 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then cd onnx git checkout ${ONNX_VERSION} git submodule update --init --recursive - pip install --no-cache-dir -r requirements.txt + # aipcc index does not have numpy>=1.22 + pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements.txt CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" export CMAKE_ARGS pip wheel . -w /root/onnx_wheel From b44381a925ca461144ec258879f11b4d3a61c4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 10:58:27 +0100 Subject: [PATCH 05/12] RHAIENG-948: fix(ppc): build failure due to missing protobuf>=4.25.1 in aipcc wheel index ``` + pip wheel . -w /root/onnx_wheel ... ERROR: Could not find a version that satisfies the requirement protobuf>=4.25.1 (from versions: none) ERROR: No matching distribution found for protobuf>=4.25.1 [end of output] ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index a055611966..16f7b3855a 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -235,7 +235,8 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements.txt CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" export CMAKE_ARGS - pip wheel . -w /root/onnx_wheel + # protobuf>=4.25.1 + pip wheel --extra-index-url https://pypi.org/simple . -w /root/onnx_wheel else echo "Skipping ONNX build on non-Power" mkdir -p /root/onnx_wheel From d91601ea8949bf261c481cbe17cbd52d4cd9dbe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 16:18:13 +0100 Subject: [PATCH 06/12] RHAIENG-948: fix(ppc): build failure due to missing numpy>=1.16.6 in aipcc wheel index ``` + pip install --no-cache-dir /tmp/wheels/pyarrow-17.0.0-cp312-cp312-linux_ppc64le.whl Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/ Processing /tmp/wheels/pyarrow-17.0.0-cp312-cp312-linux_ppc64le.whl INFO: pip is looking at multiple versions of pyarrow to determine which version is compatible with other requirements. This could take a while. ERROR: Could not find a version that satisfies the requirement numpy>=1.16.6 (from pyarrow) (from versions: none) ERROR: No matching distribution found for numpy>=1.16.6 ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index 16f7b3855a..6b4c5d53e5 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -337,7 +337,8 @@ COPY --from=pyarrow-builder /tmp/wheels /tmp/wheels RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then - pip install --no-cache-dir /tmp/wheels/*.whl + # aipcc is lacking numpy>=1.16.6 + pip install --no-cache-dir --extra-index-url https://pypi.org/simple /tmp/wheels/*.whl else echo "Skipping wheel install for $TARGETARCH" fi @@ -351,7 +352,8 @@ COPY --from=onnx-builder /root/onnx_wheel/ /onnxwheels/ RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "${TARGETARCH}" = "ppc64le" ]; then - pip install --no-cache-dir /onnxwheels/*.whl + # aipcc is sure to lack something, so add extra index preemptively + pip install --no-cache-dir --extra-index-url https://pypi.org/simple /onnxwheels/*.whl else echo "Skipping ONNX/OpenBLAS install on non-Power" fi From a8883c30286ebd4882777051977d78160825712d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 20:23:22 +0100 Subject: [PATCH 07/12] RHOAIENG-32541: chore(jupyter/datascience): add `subscription-manager refresh` step to CPU Dockerfile for base image https://issues.redhat.com/browse/RHOAIENG-32541 --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index 6b4c5d53e5..393fd1ee96 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -40,6 +40,13 @@ EOF # cpu-base # #################### FROM ${BASE_IMAGE} AS cpu-base +USER 0 +RUN /bin/bash <<'EOF' +set -Eeuxo pipefail +if command -v subscription-manager &> /dev/null; then + subscription-manager identity &>/dev/null && subscription-manager refresh || echo "Not registered, skipping refresh." +fi +EOF WORKDIR /opt/app-root/bin From e43a44ef684a5f53cc56cf926aa8e4d2f256ef63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 21:42:32 +0100 Subject: [PATCH 08/12] RHAIENG-948: fix(ppc): build failure due to missing cryptography build dependencies in aipcc wheel index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` × Failed to download and build `cryptography==43.0.3` ├─▶ Failed to resolve requirements from `build-system.requires` ├─▶ No solution found when resolving: `maturin>=1, <2`, `cffi>=1.12 │ ; platform_python_implementation != 'PyPy'`, `setuptools!=74.0.0, │ !=74.1.0, !=74.1.1, !=74.1.2, !=74.1.3, !=75.0.0, !=75.1.0, !=75.2.0` ╰─▶ Because only maturin==1.9.6 is available and maturin==1.9.6 has no wheels with a matching platform tag (e.g., `manylinux_2_34_ppc64le`), we can conclude that maturin==1.9.6 cannot be used. And because you require maturin==1.9.6, we can conclude that your requirements are unsatisfiable. hint: Wheels are available for `maturin` (v1.9.6) on the following platforms: `linux_aarch64`, `linux_x86_64` ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index 393fd1ee96..1c822722d3 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -400,10 +400,12 @@ echo "Installing software and packages" # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then # We need special flags and environment variables when building packages + # aipcc does not have some dependencies to build cryptography==43.0.3 on ppc64le, so we need to use pypi.org/simple GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \ CFLAGS="-O3" CXXFLAGS="-O3" \ uv pip install --strict --no-deps --no-cache --no-config --no-progress \ --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \ + --extra-index-url https://pypi.org/simple \ --requirements=./pylock.toml else # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, From 4bec539e00b3acad7d63bcfcd22ccf164ab20971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 23:59:48 +0100 Subject: [PATCH 09/12] RHAIENG-948: fix(ppc): build failure due to missing zlib-devel to build pillow ``` line 20, in run_setup return super().run_setup(setup_script) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/tmp/.tmpoAXecA/builds-v0/.tmpeN0QF3/lib64/python3.12/site-packages/setuptools/build_meta.py", line 317, in run_setup exec(code, locals()) File "", line 1108, in RequiredDependencyException: The headers or library files could not be found for zlib, a required dependency when compiling Pillow from source. ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index 1c822722d3..de88a6cb8d 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -386,6 +386,14 @@ else fi EOF +RUN /bin/bash <<'EOF' +set -Eeuxo pipefail +if [ "${TARGETARCH}" = "ppc64le" ]; then + # required to compile pillow + dnf install -y zlib-devel +fi +EOF + USER 1001:0 # Install Python packages and Jupyterlab extensions from pylock.toml From 4183ea4d69cf1781404bcfc487f2f1e0da0fe3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 14 Nov 2025 00:37:48 +0100 Subject: [PATCH 10/12] RHAIENG-948: fix(ppc): build failure due to missing libjpeg-turbo-devel to build pillow ``` The headers or library files could not be found for jpeg, a required dependency when compiling Pillow from source. Please see the install instructions at: https://pillow.readthedocs.io/en/latest/installation/basic-installation.html ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index de88a6cb8d..c3de1647b0 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -390,7 +390,13 @@ RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "${TARGETARCH}" = "ppc64le" ]; then # required to compile pillow - dnf install -y zlib-devel + packages=( + zlib-devel libjpeg-turbo-devel + # optional pillow deps https://pillow.readthedocs.io/en/latest/installation/building-from-source.html#external-libraries + #libtiff-devel libwebp-devel openjpeg2-devel lcms2-devel freetype-devel + #libimagequant-devel harfbuzz-devel fribidi-devel + ) + dnf install -y "${packages[@]}" fi EOF From 35944260a6638b7e02d425f806df8278a8973fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 14 Nov 2025 09:37:36 +0100 Subject: [PATCH 11/12] RHAIENG-948: fix(ppc): build failure due to missing openssl-devel to build maturin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` Could not find openssl via pkg-config: pkg-config exited with status code 1 > PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 pkg-config --libs --cflags openssl The system library `openssl` required by crate `openssl-sys` was not found. The file `openssl.pc` needs to be installed and the PKG_CONFIG_PATH environment variable must contain its parent directory. PKG_CONFIG_PATH contains the following: - /usr/local/lib/pkgconfig/ HINT: you may need to install a package such as openssl, openssl-dev or openssl-devel. ... Could not find directory of OpenSSL installation, and this `-sys` crate cannot proceed without this knowledge. If OpenSSL is installed and this crate had trouble finding it, you can set the `OPENSSL_DIR` environment variable for the compilation process. Make sure you also have the development packages of openssl installed. For example, `libssl-dev` on Ubuntu or `openssl-devel` on Fedora. ... warning: build failed, waiting for other jobs to finish... 💥 maturin failed Caused by: Failed to build a native library through cargo Caused by: Cargo build finished with "exit status: ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index c3de1647b0..f58e9efa4f 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -389,12 +389,15 @@ EOF RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "${TARGETARCH}" = "ppc64le" ]; then - # required to compile pillow packages=( + # required to compile pillow zlib-devel libjpeg-turbo-devel # optional pillow deps https://pillow.readthedocs.io/en/latest/installation/building-from-source.html#external-libraries #libtiff-devel libwebp-devel openjpeg2-devel lcms2-devel freetype-devel #libimagequant-devel harfbuzz-devel fribidi-devel + + # required to compile maturin + openssl-devel ) dnf install -y "${packages[@]}" fi From 75199d29ebcd04bb49bccf9aed7c9d9c11281348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 14 Nov 2025 10:31:33 +0100 Subject: [PATCH 12/12] RHAIENG-948: fix(ppc): build failure due to missing fortran to build scikit-learn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` × Failed to download and build `scikit-learn==1.7.2` ├─▶ Failed to install requirements from `build-system.requires` ├─▶ Failed to build `scipy==1.16.3` ├─▶ The build backend returned an error ╰─▶ Call to `mesonpy.build_wheel` failed (exit status: 1) [stdout] + meson setup ... ../meson.build:88:0: ERROR: Unknown compiler(s): [['gfortran'], ['flang-new'], ['flang'], ['nvfortran'], ['pgfortran'], ['ifort'], ['ifx'], ['g95']] The following exception(s) were encountered: Running `gfortran --help` gave "[Errno 2] No such file or directory: 'gfortran'" ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index f58e9efa4f..19ef44bf71 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -398,6 +398,9 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then # required to compile maturin openssl-devel + + # required to compile scikit-learn + gcc-gfortran ) dnf install -y "${packages[@]}" fi