Skip to content

Commit 293e091

Browse files
AaruniAggarwalAaruniAggarwal
authored
ppc64le(runtime-datascience): sync odh-io/notebooks changes into Dockerfile.konflux.cpu (#1538)
* update runtime-datascience konflux dockerfile Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com> * add packages for pillow installation failure Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com> * add packages for scipy failure Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com> * correcting package name Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com> * reverting aipcc related changes Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com> * adding pyarrow changes for ppc64le Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com> --------- Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com>
1 parent 9f5f8c4 commit 293e091

File tree

1 file changed

+258
-7
lines changed

1 file changed

+258
-7
lines changed

‎runtimes/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu‎

Lines changed: 258 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ ARG BASE_IMAGE
88
####################
99
FROM ${BASE_IMAGE} AS cpu-base
1010

11+
ARG TARGETARCH
12+
1113
WORKDIR /opt/app-root/bin
1214

1315
# OS Packages needs to be installed as root
@@ -24,7 +26,54 @@ RUN dnf -y upgrade --refresh --best --nodocs --noplugins --setopt=install_weak_d
2426
# upgrade first to avoid fixable vulnerabilities end
2527

2628
# Install useful OS packages
27-
RUN dnf install -y perl mesa-libGL skopeo libxcrypt-compat && dnf clean all && rm -rf /var/cache/yum
29+
RUN --mount=type=cache,target=/var/cache/dnf \
30+
echo "Building for architecture: ${TARGETARCH}" && \
31+
PACKAGES="perl mesa-libGL skopeo libxcrypt-compat" && \
32+
# Additional dev tools only for s390x
33+
if [ "$TARGETARCH" = "s390x" ]; then \
34+
PACKAGES="$PACKAGES gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
35+
fi && \
36+
if [ "$TARGETARCH" = "ppc64le" ]; then \
37+
PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip rust cargo unixODBC-devel cmake ninja-build"; \
38+
fi && \
39+
if [ -n "$PACKAGES" ]; then \
40+
echo "Installing: $PACKAGES" && \
41+
dnf install -y $PACKAGES && \
42+
dnf clean all && rm -rf /var/cache/yum; \
43+
fi
44+
45+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
46+
echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/' >> /etc/profile.d/ppc64le.sh && \
47+
echo 'export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH' >> /etc/profile.d/ppc64le.sh && \
48+
echo 'export OPENBLAS_VERSION=0.3.30' >> /etc/profile.d/ppc64le.sh && \
49+
echo 'export ONNX_VERSION=1.19.0' >> /etc/profile.d/ppc64le.sh && \
50+
echo 'export PYARROW_VERSION=17.0.0' >> /etc/profile.d/ppc64le.sh && \
51+
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /etc/profile.d/ppc64le.sh && \
52+
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/ppc64le.sh; \
53+
fi
54+
55+
# For s390x only, set ENV vars and install Rust
56+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
57+
# Install Rust and set up environment
58+
mkdir -p /opt/.cargo && \
59+
export HOME=/root && \
60+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh && \
61+
chmod +x rustup-init.sh && \
62+
CARGO_HOME=/opt/.cargo HOME=/root ./rustup-init.sh -y --no-modify-path && \
63+
rm -f rustup-init.sh && \
64+
chown -R 1001:0 /opt/.cargo && \
65+
# Set environment variables
66+
echo 'export PATH=/opt/.cargo/bin:$PATH' >> /etc/profile.d/cargo.sh && \
67+
echo 'export CARGO_HOME=/opt/.cargo' >> /etc/profile.d/cargo.sh && \
68+
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/cargo.sh; \
69+
fi
70+
71+
# Set python alternatives only for s390x (not needed for other arches)
72+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
73+
alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
74+
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \
75+
python --version && python3 --version; \
76+
fi
2877

2978
# Other apps and tools installed as default user
3079
USER 1001
@@ -40,28 +89,230 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
4089
rm -f /tmp/openshift-client-linux.tar.gz
4190
# Install the oc client end
4291

92+
##############################
93+
# wheel-builder stage #
94+
# NOTE: Only used in s390x
95+
##############################
96+
FROM cpu-base AS s390x-builder
97+
98+
ARG TARGETARCH
99+
USER 0
100+
WORKDIR /tmp/build-wheels
101+
102+
# Build pyarrow optimized for s390x
103+
RUN --mount=type=cache,target=/root/.cache/pip \
104+
--mount=type=cache,target=/root/.cache/dnf \
105+
if [ "$TARGETARCH" = "s390x" ]; then \
106+
# Install build dependencies (shared for pyarrow and onnx)
107+
dnf install -y cmake make gcc-c++ pybind11-devel wget && \
108+
dnf clean all && \
109+
# Build and collect pyarrow wheel
110+
git clone --depth 1 https://github.com/apache/arrow.git && \
111+
cd arrow/cpp && \
112+
mkdir release && cd release && \
113+
cmake -DCMAKE_BUILD_TYPE=Release \
114+
-DCMAKE_INSTALL_PREFIX=/usr/local \
115+
-DARROW_PYTHON=ON \
116+
-DARROW_PARQUET=ON \
117+
-DARROW_ORC=ON \
118+
-DARROW_FILESYSTEM=ON \
119+
-DARROW_JSON=ON \
120+
-DARROW_CSV=ON \
121+
-DARROW_DATASET=ON \
122+
-DARROW_DEPENDENCY_SOURCE=BUNDLED \
123+
-DARROW_WITH_LZ4=OFF \
124+
-DARROW_WITH_ZSTD=OFF \
125+
-DARROW_WITH_SNAPPY=OFF \
126+
-DARROW_BUILD_TESTS=OFF \
127+
-DARROW_BUILD_BENCHMARKS=OFF \
128+
.. && \
129+
make -j$(nproc) VERBOSE=1 && \
130+
make install -j$(nproc) && \
131+
cd ../../python && \
132+
pip install --no-cache-dir -r requirements-build.txt && \
133+
PYARROW_WITH_PARQUET=1 \
134+
PYARROW_WITH_DATASET=1 \
135+
PYARROW_WITH_FILESYSTEM=1 \
136+
PYARROW_WITH_JSON=1 \
137+
PYARROW_WITH_CSV=1 \
138+
PYARROW_PARALLEL=$(nproc) \
139+
python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
140+
mkdir -p /tmp/wheels && \
141+
cp dist/pyarrow-*.whl /tmp/wheels/ && \
142+
# Ensure wheels directory exists and has content
143+
ls -la /tmp/wheels/; \
144+
else \
145+
# Create empty wheels directory for non-s390x
146+
mkdir -p /tmp/wheels; \
147+
fi
148+
149+
###################################
150+
# openblas builder stage for ppc64le
151+
##################################
152+
153+
FROM cpu-base AS openblas-builder
154+
USER root
155+
WORKDIR /root
156+
157+
ARG TARGETARCH
158+
159+
ENV OPENBLAS_VERSION=0.3.30
160+
161+
RUN echo "openblas-builder stage TARGETARCH: ${TARGETARCH}"
162+
163+
# Download and build OpenBLAS
164+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
165+
source /opt/rh/gcc-toolset-13/enable && \
166+
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip && \
167+
unzip OpenBLAS-${OPENBLAS_VERSION}.zip && cd OpenBLAS-${OPENBLAS_VERSION} && \
168+
make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0; \
169+
else \
170+
echo "Not ppc64le, skipping OpenBLAS build" && mkdir -p /root/OpenBLAS-dummy; \
171+
fi
172+
173+
###################################
174+
# onnx builder stage for ppc64le
175+
###################################
176+
177+
FROM cpu-base AS onnx-builder
178+
USER root
179+
WORKDIR /root
180+
181+
ARG TARGETARCH
182+
ENV ONNX_VERSION=1.19.0
183+
184+
RUN echo "onnx-builder stage TARGETARCH: ${TARGETARCH}"
185+
186+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
187+
source /opt/rh/gcc-toolset-13/enable && \
188+
git clone --recursive https://github.com/onnx/onnx.git && \
189+
cd onnx && git checkout v${ONNX_VERSION} && \
190+
git submodule update --init --recursive && \
191+
pip install -r requirements.txt && \
192+
export CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" && \
193+
pip wheel . -w /onnx_wheels; \
194+
else \
195+
echo "Not ppc64le, skipping ONNX build" && mkdir -p /onnx_wheels; \
196+
fi
197+
198+
###################################
199+
# pyarrow builder stage for ppc64le
200+
##################################
201+
202+
FROM cpu-base AS arrow-builder
203+
USER root
204+
WORKDIR /root
205+
206+
ARG TARGETARCH
207+
ENV PYARROW_VERSION=17.0.0
208+
209+
RUN echo "arrow-builder stage TARGETARCH: ${TARGETARCH}"
210+
211+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
212+
git clone -b apache-arrow-${PYARROW_VERSION} https://github.com/apache/arrow.git --recursive && \
213+
cd arrow && rm -rf .git && mkdir dist && \
214+
pip3 install -r python/requirements-build.txt && \
215+
export ARROW_HOME=$(pwd)/dist && \
216+
export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH && \
217+
export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH && \
218+
export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && \
219+
export ARROW_TEST_DATA="${PWD}/testing/data" && \
220+
cmake -S cpp -B cpp/build \
221+
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
222+
-DCMAKE_BUILD_TYPE=release \
223+
-DARROW_WITH_BZ2=ON \
224+
-DARROW_WITH_ZLIB=ON \
225+
-DARROW_WITH_ZSTD=ON \
226+
-DARROW_WITH_LZ4=ON \
227+
-DARROW_WITH_SNAPPY=ON \
228+
-DARROW_WITH_BROTLI=ON \
229+
-DARROW_DATASET=ON \
230+
-DARROW_FILESYSTEM=ON \
231+
-DARROW_COMPUTE=ON \
232+
-DARROW_JSON=ON \
233+
-DARROW_CSV=ON \
234+
-DARROW_PYTHON=ON \
235+
-DARROW_PARQUET=ON \
236+
-DARROW_BUILD_SHARED=ON \
237+
-DARROW_BUILD_TESTS=OFF && \
238+
cd cpp/build && \
239+
make -j20 install && \
240+
export PYARROW_PARALLEL=20 && \
241+
export PYARROW_WITH_PARQUET=1 && \
242+
export PYARROW_WITH_DATASET=1 && \
243+
export PYARROW_BUNDLE_ARROW_CPP=1 && \
244+
pip3 install wheel && \
245+
cd ../../python && \
246+
python setup.py build_ext \
247+
--build-type=release \
248+
--bundle-arrow-cpp \
249+
bdist_wheel --dist-dir /arrowwheels; \
250+
else \
251+
echo "Not ppc64le, skipping pyarrow build" && mkdir -p /arrowwheels; \
252+
fi
43253
#######################
44254
# runtime-datascience #
45255
#######################
46256
FROM cpu-base AS runtime-datascience
47257

258+
ARG TARGETARCH
48259
ARG DATASCIENCE_SOURCE_CODE=runtimes/datascience/ubi9-python-3.12
49260

50261
WORKDIR /opt/app-root/bin
262+
USER 0
263+
264+
# Install ppc64le-built wheels if available
265+
COPY --from=openblas-builder /root/OpenBLAS-* /openblas
266+
COPY --from=onnx-builder /onnx_wheels /tmp/onnx_wheels
267+
COPY --from=arrow-builder /arrowwheels /tmp/arrowwheels
51268

52-
# Install Python packages from requirements.txt
269+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
270+
echo "Installing ppc64le ONNX, pyarrow wheels and OpenBLAS..." && \
271+
HOME=/root pip install /tmp/onnx_wheels/*.whl /tmp/arrowwheels/*.whl && \
272+
if [ -d "/openblas" ] && [ "$(ls -A /openblas 2>/dev/null)" ]; then \
273+
PREFIX=/usr/local make -C /openblas install; \
274+
fi && rm -rf /openblas /tmp/onnx_wheels /tmp/arrowwheels; \
275+
else \
276+
echo "Skipping architecture-specific wheel installs for (${TARGETARCH})" && \
277+
rm -rf /tmp/wheels /openblas /tmp/onnx_wheels /tmp/arrowwheels; \
278+
fi
279+
280+
USER 0
281+
# Copy wheels from build stage (s390x only)
282+
COPY --from=s390x-builder /tmp/wheels /tmp/wheels
283+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
284+
pip install --no-cache-dir /tmp/wheels/*.whl && rm -rf /tmp/wheels; \
285+
else \
286+
echo "Skipping wheel install for $TARGETARCH"; \
287+
fi
288+
289+
# Install Python packages from pylock.toml
53290
COPY ${DATASCIENCE_SOURCE_CODE}/pylock.toml ./
54291
# Copy Elyra dependencies for air-gapped enviroment
55292
COPY ${DATASCIENCE_SOURCE_CODE}/utils ./utils/
56293

57-
RUN echo "Installing softwares and packages" && \
58-
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
59-
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
60-
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml && \
61-
# Fix permissions to support pip in Openshift environments \
294+
RUN --mount=type=cache,target=/root/.cache/pip \
295+
echo "Installing softwares and packages" && \
296+
if [ "$TARGETARCH" = "ppc64le" ]; then \
297+
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig; \
298+
export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH && \
299+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
300+
elif [ "$TARGETARCH" = "s390x" ]; then \
301+
# For s390x, we need special flags and environment variables for building packages
302+
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
303+
CFLAGS="-O3" CXXFLAGS="-O3" \
304+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
305+
else \
306+
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
307+
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
308+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
309+
fi && \
310+
# Fix permissions to support pip in Openshift environments
62311
chmod -R g+w /opt/app-root/lib/python3.12/site-packages && \
63312
fix-permissions /opt/app-root -P
64313

314+
USER 1001
315+
65316
WORKDIR /opt/app-root/src
66317

67318
LABEL name="rhoai/odh-pipeline-runtime-datascience-cpu-py312-rhel9" \

0 commit comments

Comments
 (0)