Skip to content

Commit c05a586

Browse files
AaruniAggarwalAaruniAggarwal
committed
update runtime-datascience konflux dockerfile
Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com>
1 parent b9278f6 commit c05a586

File tree

1 file changed

+201
-7
lines changed

1 file changed

+201
-7
lines changed

‎runtimes/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu‎

Lines changed: 201 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ ARG BASE_IMAGE
88
####################
99
FROM ${BASE_IMAGE} AS cpu-base
1010

11+
ARG TARGETARCH
12+
1113
WORKDIR /opt/app-root/bin
1214

1315
# OS Packages needs to be installed as root
@@ -24,7 +26,53 @@ RUN dnf -y upgrade --refresh --best --nodocs --noplugins --setopt=install_weak_d
2426
# upgrade first to avoid fixable vulnerabilities end
2527

2628
# Install useful OS packages
27-
RUN dnf install -y perl mesa-libGL skopeo libxcrypt-compat && dnf clean all && rm -rf /var/cache/yum
29+
RUN --mount=type=cache,target=/var/cache/dnf \
30+
echo "Building for architecture: ${TARGETARCH}" && \
31+
PACKAGES="perl mesa-libGL skopeo libxcrypt-compat" && \
32+
# Additional dev tools only for s390x
33+
if [ "$TARGETARCH" = "s390x" ]; then \
34+
PACKAGES="$PACKAGES gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
35+
fi && \
36+
if [ "$TARGETARCH" = "ppc64le" ]; then \
37+
PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip unixODBC-devel cmake ninja-build"; \
38+
fi && \
39+
if [ -n "$PACKAGES" ]; then \
40+
echo "Installing: $PACKAGES" && \
41+
dnf install -y $PACKAGES && \
42+
dnf clean all && rm -rf /var/cache/yum; \
43+
fi
44+
45+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
46+
echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/' >> /etc/profile.d/ppc64le.sh && \
47+
echo 'export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH' >> /etc/profile.d/ppc64le.sh && \
48+
echo 'export OPENBLAS_VERSION=0.3.30' >> /etc/profile.d/ppc64le.sh && \
49+
echo 'export ONNX_VERSION=1.19.0' >> /etc/profile.d/ppc64le.sh && \
50+
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /etc/profile.d/ppc64le.sh && \
51+
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/ppc64le.sh; \
52+
fi
53+
54+
# For s390x only, set ENV vars and install Rust
55+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
56+
# Install Rust and set up environment
57+
mkdir -p /opt/.cargo && \
58+
export HOME=/root && \
59+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh && \
60+
chmod +x rustup-init.sh && \
61+
CARGO_HOME=/opt/.cargo HOME=/root ./rustup-init.sh -y --no-modify-path && \
62+
rm -f rustup-init.sh && \
63+
chown -R 1001:0 /opt/.cargo && \
64+
# Set environment variables
65+
echo 'export PATH=/opt/.cargo/bin:$PATH' >> /etc/profile.d/cargo.sh && \
66+
echo 'export CARGO_HOME=/opt/.cargo' >> /etc/profile.d/cargo.sh && \
67+
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/cargo.sh; \
68+
fi
69+
70+
# Set python alternatives only for s390x (not needed for other arches)
71+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
72+
alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
73+
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \
74+
python --version && python3 --version; \
75+
fi
2876

2977
# Other apps and tools installed as default user
3078
USER 1001
@@ -40,28 +88,174 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
4088
rm -f /tmp/openshift-client-linux.tar.gz
4189
# Install the oc client end
4290

91+
##############################
92+
# wheel-builder stage #
93+
# NOTE: Only used in s390x
94+
##############################
95+
FROM cpu-base AS s390x-builder
96+
97+
ARG TARGETARCH
98+
USER 0
99+
WORKDIR /tmp/build-wheels
100+
101+
# Build pyarrow optimized for s390x
102+
RUN --mount=type=cache,target=/root/.cache/pip \
103+
--mount=type=cache,target=/root/.cache/dnf \
104+
if [ "$TARGETARCH" = "s390x" ]; then \
105+
# Install build dependencies (shared for pyarrow and onnx)
106+
dnf install -y cmake make gcc-c++ pybind11-devel wget && \
107+
dnf clean all && \
108+
# Build and collect pyarrow wheel
109+
git clone --depth 1 https://github.com/apache/arrow.git && \
110+
cd arrow/cpp && \
111+
mkdir release && cd release && \
112+
cmake -DCMAKE_BUILD_TYPE=Release \
113+
-DCMAKE_INSTALL_PREFIX=/usr/local \
114+
-DARROW_PYTHON=ON \
115+
-DARROW_PARQUET=ON \
116+
-DARROW_ORC=ON \
117+
-DARROW_FILESYSTEM=ON \
118+
-DARROW_JSON=ON \
119+
-DARROW_CSV=ON \
120+
-DARROW_DATASET=ON \
121+
-DARROW_DEPENDENCY_SOURCE=BUNDLED \
122+
-DARROW_WITH_LZ4=OFF \
123+
-DARROW_WITH_ZSTD=OFF \
124+
-DARROW_WITH_SNAPPY=OFF \
125+
-DARROW_BUILD_TESTS=OFF \
126+
-DARROW_BUILD_BENCHMARKS=OFF \
127+
.. && \
128+
make -j$(nproc) VERBOSE=1 && \
129+
make install -j$(nproc) && \
130+
cd ../../python && \
131+
pip install --no-cache-dir -r requirements-build.txt && \
132+
PYARROW_WITH_PARQUET=1 \
133+
PYARROW_WITH_DATASET=1 \
134+
PYARROW_WITH_FILESYSTEM=1 \
135+
PYARROW_WITH_JSON=1 \
136+
PYARROW_WITH_CSV=1 \
137+
PYARROW_PARALLEL=$(nproc) \
138+
python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
139+
mkdir -p /tmp/wheels && \
140+
cp dist/pyarrow-*.whl /tmp/wheels/ && \
141+
# Ensure wheels directory exists and has content
142+
ls -la /tmp/wheels/; \
143+
else \
144+
# Create empty wheels directory for non-s390x
145+
mkdir -p /tmp/wheels; \
146+
fi
147+
148+
###################################
149+
# openblas builder stage for ppc64le
150+
##################################
151+
152+
FROM cpu-base AS openblas-builder
153+
USER root
154+
WORKDIR /root
155+
156+
ARG TARGETARCH
157+
158+
ENV OPENBLAS_VERSION=0.3.30
159+
160+
RUN echo "openblas-builder stage TARGETARCH: ${TARGETARCH}"
161+
162+
# Download and build OpenBLAS
163+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
164+
source /opt/rh/gcc-toolset-13/enable && \
165+
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip && \
166+
unzip OpenBLAS-${OPENBLAS_VERSION}.zip && cd OpenBLAS-${OPENBLAS_VERSION} && \
167+
make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0; \
168+
else \
169+
echo "Not ppc64le, skipping OpenBLAS build" && mkdir -p /root/OpenBLAS-dummy; \
170+
fi
171+
172+
###################################
173+
# onnx builder stage for ppc64le
174+
###################################
175+
176+
FROM cpu-base AS onnx-builder
177+
USER root
178+
WORKDIR /root
179+
180+
ARG TARGETARCH
181+
ENV ONNX_VERSION=1.19.0
182+
183+
RUN echo "onnx-builder stage TARGETARCH: ${TARGETARCH}"
184+
185+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
186+
source /opt/rh/gcc-toolset-13/enable && \
187+
git clone --recursive https://github.com/onnx/onnx.git && \
188+
cd onnx && git checkout v${ONNX_VERSION} && \
189+
git submodule update --init --recursive && \
190+
pip install -r requirements.txt && \
191+
export CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" && \
192+
pip wheel . -w /onnx_wheels; \
193+
else \
194+
echo "Not ppc64le, skipping ONNX build" && mkdir -p /onnx_wheels; \
195+
fi
196+
43197
#######################
44198
# runtime-datascience #
45199
#######################
46200
FROM cpu-base AS runtime-datascience
47201

202+
ARG TARGETARCH
48203
ARG DATASCIENCE_SOURCE_CODE=runtimes/datascience/ubi9-python-3.12
49204

50205
WORKDIR /opt/app-root/bin
206+
USER 0
51207

52-
# Install Python packages from requirements.txt
208+
# Install ppc64le-built wheels if available
209+
COPY --from=openblas-builder /root/OpenBLAS-* /openblas
210+
COPY --from=onnx-builder /onnx_wheels /tmp/onnx_wheels
211+
212+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
213+
echo "Installing ppc64le ONNX wheels and OpenBLAS..." && \
214+
HOME=/root pip install /tmp/onnx_wheels/*.whl && \
215+
if [ -d "/openblas" ] && [ "$(ls -A /openblas 2>/dev/null)" ]; then \
216+
PREFIX=/usr/local make -C /openblas install; \
217+
fi && rm -rf /openblas /tmp/onnx_wheels; \
218+
else \
219+
echo "Skipping architecture-specific wheel installs for (${TARGETARCH})" && \
220+
rm -rf /tmp/wheels /openblas /tmp/onnx_wheels; \
221+
fi
222+
223+
USER 0
224+
# Copy wheels from build stage (s390x only)
225+
COPY --from=s390x-builder /tmp/wheels /tmp/wheels
226+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
227+
pip install --no-cache-dir /tmp/wheels/*.whl && rm -rf /tmp/wheels; \
228+
else \
229+
echo "Skipping wheel install for $TARGETARCH"; \
230+
fi
231+
232+
# Install Python packages from pylock.toml
53233
COPY ${DATASCIENCE_SOURCE_CODE}/pylock.toml ./
54234
# Copy Elyra dependencies for air-gapped enviroment
55235
COPY ${DATASCIENCE_SOURCE_CODE}/utils ./utils/
56236

57-
RUN echo "Installing softwares and packages" && \
58-
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
59-
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
60-
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml && \
61-
# Fix permissions to support pip in Openshift environments \
237+
RUN --mount=type=cache,target=/root/.cache/pip \
238+
echo "Installing softwares and packages" && \
239+
if [ "$TARGETARCH" = "ppc64le" ]; then \
240+
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig; \
241+
export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH && \
242+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
243+
elif [ "$TARGETARCH" = "s390x" ]; then \
244+
# For s390x, we need special flags and environment variables for building packages
245+
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
246+
CFLAGS="-O3" CXXFLAGS="-O3" \
247+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
248+
else \
249+
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
250+
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
251+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
252+
fi && \
253+
# Fix permissions to support pip in Openshift environments
62254
chmod -R g+w /opt/app-root/lib/python3.12/site-packages && \
63255
fix-permissions /opt/app-root -P
64256

257+
USER 1001
258+
65259
WORKDIR /opt/app-root/src
66260

67261
LABEL name="rhoai/odh-pipeline-runtime-datascience-cpu-py312-rhel9" \

0 commit comments

Comments
 (0)