From 4c0e07f544777f78187c038011f447ede7cb5e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 08:17:48 +0100 Subject: [PATCH 01/12] RHAIENG-948: fix(ppc): build failure due to missing curl add `libcurl-devel` to build dependencies for `ppc64le` and `s390x` in pyarrow and onnx builds ``` -- Building AWS C++ SDK from source CMake Error at /usr/share/cmake/Modules/FindPackageHandleStandardArgs.cmake:230 (message): Could NOT find CURL (missing: CURL_LIBRARY CURL_INCLUDE_DIR) ``` # Conflicts: # jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu --- .../datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 8e5018b307..ce70e7fddf 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -146,7 +146,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \ set -Eeuxo pipefail if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then # Install build dependencies (shared for pyarrow and onnx) - dnf install -y cmake make gcc-c++ pybind11-devel wget + packages=( + cmake make gcc-c++ pybind11-devel wget + # pyarrow -DARROW_S3=ON pulls in AWS C++ SDK + # libcurl-devel is required to build AWS C++ SDK + libcurl-devel + ) + dnf install -y "${packages[@]}" dnf clean all # Build and collect pyarrow wheel git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git From 97a2586f6e35487f84155a515c5300c90e7d5dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 08:35:03 +0100 Subject: [PATCH 02/12] RHAIENG-948: fix(ppc): build failure due to missing openssl add `libcurl-devel` to build dependencies for `ppc64le` and `s390x` in pyarrow and onnx builds ``` -- Configuring done (1.8s) -- Generating done (0.1s) CMake Error at cmake_modules/ThirdpartyToolchain.cmake:5282 (set_property): The link interface of target "AWS::aws-c-cal" contains: OpenSSL::Crypto but the target was not found. Possible reasons include: * There is a typo in the target name. * A find_package call is missing for an IMPORTED target. * An ALIAS target is missing. ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index ce70e7fddf..6ec30c0ad3 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -150,7 +150,7 @@ if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then cmake make gcc-c++ pybind11-devel wget # pyarrow -DARROW_S3=ON pulls in AWS C++ SDK # libcurl-devel is required to build AWS C++ SDK - libcurl-devel + libcurl-devel openssl-devel ) dnf install -y "${packages[@]}" dnf clean all From 60883d9d98713e6b1394fae82f90e5c4a0772c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 08:50:56 +0100 Subject: [PATCH 03/12] RHAIENG-948: fix(ppc): build failure due to missing cython in aipcc wheel index ``` + pip install --no-cache-dir -r requirements-build.txt Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/ Ignoring oldest-supported-numpy: markers 'python_version < "3.9"' don't match your environment ERROR: Could not find a version that satisfies the requirement cython>=0.29.31 (from versions: none) ERROR: No matching distribution found for cython>=0.29.31 ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 6ec30c0ad3..906d1f6928 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -181,7 +181,8 @@ if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then make -j$(nproc) VERBOSE=1 make install -j$(nproc) cd ../../python - pip install --no-cache-dir -r requirements-build.txt + # aipcc index is missing cython, and maybe more + pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements-build.txt PYARROW_WITH_PARQUET=1 \ PYARROW_WITH_DATASET=1 \ PYARROW_WITH_FILESYSTEM=1 \ From 245425fc3bbbfc4bd035c399e97c7beb1f72c0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 09:35:40 +0100 Subject: [PATCH 04/12] RHAIENG-948: fix(ppc): build failure due to missing numpy in aipcc wheel index ``` + pip install --no-cache-dir -r requirements.txt Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/ ERROR: Could not find a version that satisfies the requirement numpy>=1.22 (from versions: none) ERROR: No matching distribution found for numpy>=1.22 subprocess exited with status 1 ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 906d1f6928..0f6b5970b8 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -233,7 +233,8 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then cd onnx git checkout ${ONNX_VERSION} git submodule update --init --recursive - pip install --no-cache-dir -r requirements.txt + # aipcc index does not have numpy>=1.22 + pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements.txt CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" export CMAKE_ARGS pip wheel . -w /root/onnx_wheel From 2e773866edcf261f8e97707ce0f40fe2a8407be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 10:58:27 +0100 Subject: [PATCH 05/12] RHAIENG-948: fix(ppc): build failure due to missing protobuf>=4.25.1 in aipcc wheel index ``` + pip wheel . -w /root/onnx_wheel ... ERROR: Could not find a version that satisfies the requirement protobuf>=4.25.1 (from versions: none) ERROR: No matching distribution found for protobuf>=4.25.1 [end of output] ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 0f6b5970b8..d54ca52e31 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -237,7 +237,8 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements.txt CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" export CMAKE_ARGS - pip wheel . -w /root/onnx_wheel + # protobuf>=4.25.1 + pip wheel --extra-index-url https://pypi.org/simple . -w /root/onnx_wheel else echo "Skipping ONNX build on non-Power" mkdir -p /root/onnx_wheel From 724b247fb6b62262fc8bea42ef00eb3392042c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 16:18:13 +0100 Subject: [PATCH 06/12] RHAIENG-948: fix(ppc): build failure due to missing numpy>=1.16.6 in aipcc wheel index ``` + pip install --no-cache-dir /tmp/wheels/pyarrow-17.0.0-cp312-cp312-linux_ppc64le.whl Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/ Processing /tmp/wheels/pyarrow-17.0.0-cp312-cp312-linux_ppc64le.whl INFO: pip is looking at multiple versions of pyarrow to determine which version is compatible with other requirements. This could take a while. ERROR: Could not find a version that satisfies the requirement numpy>=1.16.6 (from pyarrow) (from versions: none) ERROR: No matching distribution found for numpy>=1.16.6 ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index d54ca52e31..9a05ae50b8 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -337,7 +337,8 @@ COPY --from=pyarrow-builder /tmp/wheels /tmp/wheels RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then - pip install --no-cache-dir /tmp/wheels/*.whl + # aipcc is lacking numpy>=1.16.6 + pip install --no-cache-dir --extra-index-url https://pypi.org/simple /tmp/wheels/*.whl else echo "Skipping wheel install for $TARGETARCH" fi @@ -351,7 +352,8 @@ COPY --from=onnx-builder /root/onnx_wheel/ /onnxwheels/ RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "${TARGETARCH}" = "ppc64le" ]; then - pip install --no-cache-dir /onnxwheels/*.whl + # aipcc is sure to lack something, so add extra index preemptively + pip install --no-cache-dir --extra-index-url https://pypi.org/simple /onnxwheels/*.whl else echo "Skipping ONNX/OpenBLAS install on non-Power" fi From ee8efc2fb57af68b225c519e699f0ad3953e4fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 18:03:45 +0100 Subject: [PATCH 07/12] Add `subscription-manager register` step for base image initialization --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 9a05ae50b8..fdfbe4ba31 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -41,6 +41,7 @@ EOF #################### FROM ${BASE_IMAGE} AS cpu-base USER 0 +RUN subscription-manager register --org 18631088 --activationkey thisisunsafe RUN subscription-manager refresh WORKDIR /opt/app-root/bin From 7077f4a1535bc9f49f7140c9bbf55f3bc32f3bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 21:42:32 +0100 Subject: [PATCH 08/12] RHAIENG-948: fix(ppc): build failure due to missing cryptography build dependencies in aipcc wheel index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` × Failed to download and build `cryptography==43.0.3` ├─▶ Failed to resolve requirements from `build-system.requires` ├─▶ No solution found when resolving: `maturin>=1, <2`, `cffi>=1.12 │ ; platform_python_implementation != 'PyPy'`, `setuptools!=74.0.0, │ !=74.1.0, !=74.1.1, !=74.1.2, !=74.1.3, !=75.0.0, !=75.1.0, !=75.2.0` ╰─▶ Because only maturin==1.9.6 is available and maturin==1.9.6 has no wheels with a matching platform tag (e.g., `manylinux_2_34_ppc64le`), we can conclude that maturin==1.9.6 cannot be used. And because you require maturin==1.9.6, we can conclude that your requirements are unsatisfiable. hint: Wheels are available for `maturin` (v1.9.6) on the following platforms: `linux_aarch64`, `linux_x86_64` ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index fdfbe4ba31..02120eb819 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -394,10 +394,12 @@ echo "Installing software and packages" # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then # We need special flags and environment variables when building packages + # aipcc does not have some dependencies to build cryptography==43.0.3 on ppc64le, so we need to use pypi.org/simple GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \ CFLAGS="-O3" CXXFLAGS="-O3" \ uv pip install --strict --no-deps --no-cache --no-config --no-progress \ --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \ + --extra-index-url https://pypi.org/simple \ --requirements=./pylock.toml else # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, From 3c76a4c4a1f0496734fe2580003986d5fe0178c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Thu, 13 Nov 2025 23:59:48 +0100 Subject: [PATCH 09/12] RHAIENG-948: fix(ppc): build failure due to missing zlib-devel to build pillow ``` line 20, in run_setup return super().run_setup(setup_script) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/tmp/.tmpoAXecA/builds-v0/.tmpeN0QF3/lib64/python3.12/site-packages/setuptools/build_meta.py", line 317, in run_setup exec(code, locals()) File "", line 1108, in RequiredDependencyException: The headers or library files could not be found for zlib, a required dependency when compiling Pillow from source. ``` --- .../datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 02120eb819..877d4797ec 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -380,6 +380,14 @@ else fi EOF +RUN /bin/bash <<'EOF' +set -Eeuxo pipefail +if [ "${TARGETARCH}" = "ppc64le" ]; then + # required to compile pillow + dnf install -y zlib-devel +fi +EOF + USER 1001:0 # Install Python packages and Jupyterlab extensions from pylock.toml From e5bf561f6e81edd8a762c57bd5f3135905739d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 14 Nov 2025 00:37:48 +0100 Subject: [PATCH 10/12] RHAIENG-948: fix(ppc): build failure due to missing libjpeg-turbo-devel to build pillow ``` The headers or library files could not be found for jpeg, a required dependency when compiling Pillow from source. Please see the install instructions at: https://pillow.readthedocs.io/en/latest/installation/basic-installation.html ``` --- .../datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 877d4797ec..d74bba0662 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -384,7 +384,13 @@ RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "${TARGETARCH}" = "ppc64le" ]; then # required to compile pillow - dnf install -y zlib-devel + packages=( + zlib-devel libjpeg-turbo-devel + # optional pillow deps https://pillow.readthedocs.io/en/latest/installation/building-from-source.html#external-libraries + #libtiff-devel libwebp-devel openjpeg2-devel lcms2-devel freetype-devel + #libimagequant-devel harfbuzz-devel fribidi-devel + ) + dnf install -y "${packages[@]}" fi EOF From 01a20475aafb9c8bce13a4302f63c0af5778f4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 14 Nov 2025 09:37:36 +0100 Subject: [PATCH 11/12] RHAIENG-948: fix(ppc): build failure due to missing openssl-devel to build maturin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` Could not find openssl via pkg-config: pkg-config exited with status code 1 > PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 pkg-config --libs --cflags openssl The system library `openssl` required by crate `openssl-sys` was not found. The file `openssl.pc` needs to be installed and the PKG_CONFIG_PATH environment variable must contain its parent directory. PKG_CONFIG_PATH contains the following: - /usr/local/lib/pkgconfig/ HINT: you may need to install a package such as openssl, openssl-dev or openssl-devel. ... Could not find directory of OpenSSL installation, and this `-sys` crate cannot proceed without this knowledge. If OpenSSL is installed and this crate had trouble finding it, you can set the `OPENSSL_DIR` environment variable for the compilation process. Make sure you also have the development packages of openssl installed. For example, `libssl-dev` on Ubuntu or `openssl-devel` on Fedora. ... warning: build failed, waiting for other jobs to finish... 💥 maturin failed Caused by: Failed to build a native library through cargo Caused by: Cargo build finished with "exit status: ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index d74bba0662..8a84eb9ead 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -383,12 +383,15 @@ EOF RUN /bin/bash <<'EOF' set -Eeuxo pipefail if [ "${TARGETARCH}" = "ppc64le" ]; then - # required to compile pillow packages=( + # required to compile pillow zlib-devel libjpeg-turbo-devel # optional pillow deps https://pillow.readthedocs.io/en/latest/installation/building-from-source.html#external-libraries #libtiff-devel libwebp-devel openjpeg2-devel lcms2-devel freetype-devel #libimagequant-devel harfbuzz-devel fribidi-devel + + # required to compile maturin + openssl-devel ) dnf install -y "${packages[@]}" fi From 2eed737094894b1759f5426b2302aaa451e62b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 14 Nov 2025 10:31:33 +0100 Subject: [PATCH 12/12] RHAIENG-948: fix(ppc): build failure due to missing fortran to build scikit-learn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` × Failed to download and build `scikit-learn==1.7.2` ├─▶ Failed to install requirements from `build-system.requires` ├─▶ Failed to build `scipy==1.16.3` ├─▶ The build backend returned an error ╰─▶ Call to `mesonpy.build_wheel` failed (exit status: 1) [stdout] + meson setup ... ../meson.build:88:0: ERROR: Unknown compiler(s): [['gfortran'], ['flang-new'], ['flang'], ['nvfortran'], ['pgfortran'], ['ifort'], ['ifx'], ['g95']] The following exception(s) were encountered: Running `gfortran --help` gave "[Errno 2] No such file or directory: 'gfortran'" ``` --- jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu index 8a84eb9ead..bd67290e8c 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -392,6 +392,9 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then # required to compile maturin openssl-devel + + # required to compile scikit-learn + gcc-gfortran ) dnf install -y "${packages[@]}" fi