Skip to content

Commit 70aed95

Browse files
AaruniAggarwalAaruniAggarwal
committed
update runtime-datascience konflux dockerfile
Signed-off-by: Aaruni Aggarwal <aaruniagg@gmail.com>
1 parent 93c7a65 commit 70aed95

File tree

1 file changed

+201
-7
lines changed

1 file changed

+201
-7
lines changed

‎runtimes/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu‎

Lines changed: 201 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ ARG BASE_IMAGE
88
####################
99
FROM ${BASE_IMAGE} AS cpu-base
1010

11+
ARG TARGETARCH
12+
1113
WORKDIR /opt/app-root/bin
1214

1315
# OS Packages needs to be installed as root
@@ -19,7 +21,53 @@ RUN dnf -y upgrade --refresh --best --nodocs --noplugins --setopt=install_weak_d
1921
# upgrade first to avoid fixable vulnerabilities end
2022

2123
# Install useful OS packages
22-
RUN dnf install -y mesa-libGL skopeo libxcrypt-compat && dnf clean all && rm -rf /var/cache/yum
24+
RUN --mount=type=cache,target=/var/cache/dnf \
25+
echo "Building for architecture: ${TARGETARCH}" && \
26+
PACKAGES="mesa-libGL skopeo libxcrypt-compat" && \
27+
# Additional dev tools only for s390x
28+
if [ "$TARGETARCH" = "s390x" ]; then \
29+
PACKAGES="$PACKAGES gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
30+
fi && \
31+
if [ "$TARGETARCH" = "ppc64le" ]; then \
32+
PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip unixODBC-devel cmake ninja-build"; \
33+
fi && \
34+
if [ -n "$PACKAGES" ]; then \
35+
echo "Installing: $PACKAGES" && \
36+
dnf install -y $PACKAGES && \
37+
dnf clean all && rm -rf /var/cache/yum; \
38+
fi
39+
40+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
41+
echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/' >> /etc/profile.d/ppc64le.sh && \
42+
echo 'export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH' >> /etc/profile.d/ppc64le.sh && \
43+
echo 'export OPENBLAS_VERSION=0.3.30' >> /etc/profile.d/ppc64le.sh && \
44+
echo 'export ONNX_VERSION=1.19.0' >> /etc/profile.d/ppc64le.sh && \
45+
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /etc/profile.d/ppc64le.sh && \
46+
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/ppc64le.sh; \
47+
fi
48+
49+
# For s390x only, set ENV vars and install Rust
50+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
51+
# Install Rust and set up environment
52+
mkdir -p /opt/.cargo && \
53+
export HOME=/root && \
54+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh && \
55+
chmod +x rustup-init.sh && \
56+
CARGO_HOME=/opt/.cargo HOME=/root ./rustup-init.sh -y --no-modify-path && \
57+
rm -f rustup-init.sh && \
58+
chown -R 1001:0 /opt/.cargo && \
59+
# Set environment variables
60+
echo 'export PATH=/opt/.cargo/bin:$PATH' >> /etc/profile.d/cargo.sh && \
61+
echo 'export CARGO_HOME=/opt/.cargo' >> /etc/profile.d/cargo.sh && \
62+
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/cargo.sh; \
63+
fi
64+
65+
# Set python alternatives only for s390x (not needed for other arches)
66+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
67+
alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
68+
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \
69+
python --version && python3 --version; \
70+
fi
2371

2472
# Other apps and tools installed as default user
2573
USER 1001
@@ -35,28 +83,174 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
3583
rm -f /tmp/openshift-client-linux.tar.gz
3684
# Install the oc client end
3785

86+
##############################
87+
# wheel-builder stage #
88+
# NOTE: Only used in s390x
89+
##############################
90+
FROM cpu-base AS s390x-builder
91+
92+
ARG TARGETARCH
93+
USER 0
94+
WORKDIR /tmp/build-wheels
95+
96+
# Build pyarrow optimized for s390x
97+
RUN --mount=type=cache,target=/root/.cache/pip \
98+
--mount=type=cache,target=/root/.cache/dnf \
99+
if [ "$TARGETARCH" = "s390x" ]; then \
100+
# Install build dependencies (shared for pyarrow and onnx)
101+
dnf install -y cmake make gcc-c++ pybind11-devel wget && \
102+
dnf clean all && \
103+
# Build and collect pyarrow wheel
104+
git clone --depth 1 https://github.com/apache/arrow.git && \
105+
cd arrow/cpp && \
106+
mkdir release && cd release && \
107+
cmake -DCMAKE_BUILD_TYPE=Release \
108+
-DCMAKE_INSTALL_PREFIX=/usr/local \
109+
-DARROW_PYTHON=ON \
110+
-DARROW_PARQUET=ON \
111+
-DARROW_ORC=ON \
112+
-DARROW_FILESYSTEM=ON \
113+
-DARROW_JSON=ON \
114+
-DARROW_CSV=ON \
115+
-DARROW_DATASET=ON \
116+
-DARROW_DEPENDENCY_SOURCE=BUNDLED \
117+
-DARROW_WITH_LZ4=OFF \
118+
-DARROW_WITH_ZSTD=OFF \
119+
-DARROW_WITH_SNAPPY=OFF \
120+
-DARROW_BUILD_TESTS=OFF \
121+
-DARROW_BUILD_BENCHMARKS=OFF \
122+
.. && \
123+
make -j$(nproc) VERBOSE=1 && \
124+
make install -j$(nproc) && \
125+
cd ../../python && \
126+
pip install --no-cache-dir -r requirements-build.txt && \
127+
PYARROW_WITH_PARQUET=1 \
128+
PYARROW_WITH_DATASET=1 \
129+
PYARROW_WITH_FILESYSTEM=1 \
130+
PYARROW_WITH_JSON=1 \
131+
PYARROW_WITH_CSV=1 \
132+
PYARROW_PARALLEL=$(nproc) \
133+
python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
134+
mkdir -p /tmp/wheels && \
135+
cp dist/pyarrow-*.whl /tmp/wheels/ && \
136+
# Ensure wheels directory exists and has content
137+
ls -la /tmp/wheels/; \
138+
else \
139+
# Create empty wheels directory for non-s390x
140+
mkdir -p /tmp/wheels; \
141+
fi
142+
143+
###################################
144+
# openblas builder stage for ppc64le
145+
##################################
146+
147+
FROM cpu-base AS openblas-builder
148+
USER root
149+
WORKDIR /root
150+
151+
ARG TARGETARCH
152+
153+
ENV OPENBLAS_VERSION=0.3.30
154+
155+
RUN echo "openblas-builder stage TARGETARCH: ${TARGETARCH}"
156+
157+
# Download and build OpenBLAS
158+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
159+
source /opt/rh/gcc-toolset-13/enable && \
160+
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip && \
161+
unzip OpenBLAS-${OPENBLAS_VERSION}.zip && cd OpenBLAS-${OPENBLAS_VERSION} && \
162+
make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0; \
163+
else \
164+
echo "Not ppc64le, skipping OpenBLAS build" && mkdir -p /root/OpenBLAS-dummy; \
165+
fi
166+
167+
###################################
168+
# onnx builder stage for ppc64le
169+
###################################
170+
171+
FROM cpu-base AS onnx-builder
172+
USER root
173+
WORKDIR /root
174+
175+
ARG TARGETARCH
176+
ENV ONNX_VERSION=1.19.0
177+
178+
RUN echo "onnx-builder stage TARGETARCH: ${TARGETARCH}"
179+
180+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
181+
source /opt/rh/gcc-toolset-13/enable && \
182+
git clone --recursive https://github.com/onnx/onnx.git && \
183+
cd onnx && git checkout v${ONNX_VERSION} && \
184+
git submodule update --init --recursive && \
185+
pip install -r requirements.txt && \
186+
export CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)" && \
187+
pip wheel . -w /onnx_wheels; \
188+
else \
189+
echo "Not ppc64le, skipping ONNX build" && mkdir -p /onnx_wheels; \
190+
fi
191+
38192
#######################
39193
# runtime-datascience #
40194
#######################
41195
FROM cpu-base AS runtime-datascience
42196

197+
ARG TARGETARCH
43198
ARG DATASCIENCE_SOURCE_CODE=runtimes/datascience/ubi9-python-3.12
44199

45200
WORKDIR /opt/app-root/bin
201+
USER 0
46202

47-
# Install Python packages from requirements.txt
203+
# Install ppc64le-built wheels if available
204+
COPY --from=openblas-builder /root/OpenBLAS-* /openblas
205+
COPY --from=onnx-builder /onnx_wheels /tmp/onnx_wheels
206+
207+
RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
208+
echo "Installing ppc64le ONNX wheels and OpenBLAS..." && \
209+
HOME=/root pip install /tmp/onnx_wheels/*.whl && \
210+
if [ -d "/openblas" ] && [ "$(ls -A /openblas 2>/dev/null)" ]; then \
211+
PREFIX=/usr/local make -C /openblas install; \
212+
fi && rm -rf /openblas /tmp/onnx_wheels; \
213+
else \
214+
echo "Skipping architecture-specific wheel installs for (${TARGETARCH})" && \
215+
rm -rf /tmp/wheels /openblas /tmp/onnx_wheels; \
216+
fi
217+
218+
USER 0
219+
# Copy wheels from build stage (s390x only)
220+
COPY --from=s390x-builder /tmp/wheels /tmp/wheels
221+
RUN if [ "$TARGETARCH" = "s390x" ]; then \
222+
pip install --no-cache-dir /tmp/wheels/*.whl && rm -rf /tmp/wheels; \
223+
else \
224+
echo "Skipping wheel install for $TARGETARCH"; \
225+
fi
226+
227+
# Install Python packages from pylock.toml
48228
COPY ${DATASCIENCE_SOURCE_CODE}/pylock.toml ./
49229
# Copy Elyra dependencies for air-gapped enviroment
50230
COPY ${DATASCIENCE_SOURCE_CODE}/utils ./utils/
51231

52-
RUN echo "Installing softwares and packages" && \
53-
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
54-
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
55-
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml && \
56-
# Fix permissions to support pip in Openshift environments \
232+
RUN --mount=type=cache,target=/root/.cache/pip \
233+
echo "Installing softwares and packages" && \
234+
if [ "$TARGETARCH" = "ppc64le" ]; then \
235+
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig; \
236+
export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH && \
237+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
238+
elif [ "$TARGETARCH" = "s390x" ]; then \
239+
# For s390x, we need special flags and environment variables for building packages
240+
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
241+
CFLAGS="-O3" CXXFLAGS="-O3" \
242+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
243+
else \
244+
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
245+
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
246+
uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml; \
247+
fi && \
248+
# Fix permissions to support pip in Openshift environments
57249
chmod -R g+w /opt/app-root/lib/python3.12/site-packages && \
58250
fix-permissions /opt/app-root -P
59251

252+
USER 1001
253+
60254
WORKDIR /opt/app-root/src
61255

62256
LABEL name="rhoai/odh-pipeline-runtime-datascience-cpu-py312-rhel9" \

0 commit comments

Comments
 (0)