diff --git a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu index ab8868f176..2902d381a7 100644 --- a/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu @@ -19,19 +19,22 @@ ARG TARGETARCH # Keep s390x special-case from original (create dummy binary) but # include explicit curl/unzip steps from the delta for non-s390x. -RUN arch="${TARGETARCH:-$(uname -m)}" && \ - arch=$(echo "$arch" | cut -d- -f1) && \ - if [ "$arch" = "s390x" ]; then \ - echo "Skipping mongocli build for ${arch}, creating dummy binary"; \ - mkdir -p /tmp && printf '#!/bin/sh\necho "mongocli not supported on s390x"\n' > /tmp/mongocli && \ - chmod +x /tmp/mongocli; \ - else \ - echo "Building mongocli for ${arch}"; \ - curl -Lo mongodb-cli-mongocli-v${MONGOCLI_VERSION}.zip https://github.com/mongodb/mongodb-cli/archive/refs/tags/mongocli/v${MONGOCLI_VERSION}.zip && \ - unzip ./mongodb-cli-mongocli-v${MONGOCLI_VERSION}.zip && \ - cd ./mongodb-cli-mongocli-v${MONGOCLI_VERSION}/ && \ - CGO_ENABLED=1 GOOS=linux GOARCH=${arch} GO111MODULE=on go build -a -tags strictfipsruntime -o /tmp/mongocli ./cmd/mongocli/; \ - fi +RUN /bin/bash <<'EOF' +set -Eeuxo pipefail +arch="${TARGETARCH:-$(uname -m)}" +arch=$(echo "$arch" | cut -d- -f1) +if [ "$arch" = "s390x" ]; then + echo "Skipping mongocli build for ${arch}, creating dummy binary" + mkdir -p /tmp && printf '#!/bin/sh\necho "mongocli not supported on s390x"\n' > /tmp/mongocli + chmod +x /tmp/mongocli +else + echo "Building mongocli for ${arch}" + curl -Lo mongodb-cli-mongocli-v${MONGOCLI_VERSION}.zip https://github.com/mongodb/mongodb-cli/archive/refs/tags/mongocli/v${MONGOCLI_VERSION}.zip + unzip ./mongodb-cli-mongocli-v${MONGOCLI_VERSION}.zip + cd ./mongodb-cli-mongocli-v${MONGOCLI_VERSION}/ + CGO_ENABLED=1 GOOS=linux GOARCH=${arch} GO111MODULE=on go build -a -tags strictfipsruntime -o /tmp/mongocli ./cmd/mongocli/ +fi +EOF #################### # cpu-base # @@ -62,16 +65,19 @@ EOF # upgrade first to avoid fixable vulnerabilities end # Install useful OS packages -RUN --mount=type=cache,target=/var/cache/dnf \ - echo "Building for architecture: ${TARGETARCH}" && \ - if [ "$TARGETARCH" = "s390x" ]; then \ - PACKAGES="perl mesa-libGL skopeo gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel"; \ - else \ - PACKAGES="perl mesa-libGL skopeo"; \ - fi && \ - echo "Installing: $PACKAGES" && \ - dnf install -y $PACKAGES && \ - dnf clean all && rm -rf /var/cache/yum +RUN --mount=type=cache,target=/var/cache/dnf /bin/bash <<'EOF' +set -Eeuxo pipefail +echo "Building for architecture: ${TARGETARCH}" +if [ "$TARGETARCH" = "s390x" ]; then + PACKAGES="perl mesa-libGL skopeo gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel" +else + PACKAGES="perl mesa-libGL skopeo" +fi +echo "Installing: $PACKAGES" +dnf install -y $PACKAGES +dnf clean all +rm -rf /var/cache/yum +EOF RUN /bin/bash <<'EOF' set -Eeuxo pipefail @@ -134,55 +140,57 @@ WORKDIR /tmp/build-wheels # Build pyarrow on ppc64le and s390x RUN --mount=type=cache,target=/root/.cache/pip \ - --mount=type=cache,target=/root/.cache/dnf \ - if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \ - # Install build dependencies (shared for pyarrow and onnx) - dnf install -y cmake make gcc-c++ pybind11-devel wget && \ - dnf clean all && \ - # Build and collect pyarrow wheel - git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git && \ - cd arrow/cpp && \ - mkdir release && cd release && \ - ARROW_S3_FLAG="" && \ - if [ "$TARGETARCH" != "s390x" ]; then ARROW_S3_FLAG="-DARROW_S3=ON"; fi && \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DARROW_PYTHON=ON \ - -DARROW_PARQUET=ON \ - -DARROW_ORC=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_JSON=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ - -DARROW_WITH_LZ4=OFF \ - -DARROW_WITH_ZSTD=OFF \ - -DARROW_WITH_SNAPPY=OFF \ - ${ARROW_S3_FLAG} \ - -DARROW_SUBSTRAIT=ON \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_BUILD_BENCHMARKS=OFF \ - .. && \ - make -j$(nproc) VERBOSE=1 && \ - make install -j$(nproc) && \ - cd ../../python && \ - pip install --no-cache-dir -r requirements-build.txt && \ - PYARROW_WITH_PARQUET=1 \ - PYARROW_WITH_DATASET=1 \ - PYARROW_WITH_FILESYSTEM=1 \ - PYARROW_WITH_JSON=1 \ - PYARROW_WITH_CSV=1 \ - PYARROW_PARALLEL=$(nproc) \ - python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \ - mkdir -p /tmp/wheels && \ - cp dist/pyarrow-*.whl /tmp/wheels/ && \ - chmod -R 777 /tmp/wheels && \ - # Ensure wheels directory exists and has content - ls -la /tmp/wheels/; \ - else \ - # Create empty wheels directory for non-s390x - mkdir -p /tmp/wheels; \ - fi + --mount=type=cache,target=/root/.cache/dnf /bin/bash <<'EOF' +set -Eeuxo pipefail +if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then + # Install build dependencies (shared for pyarrow and onnx) + dnf install -y cmake make gcc-c++ pybind11-devel wget + dnf clean all + # Build and collect pyarrow wheel + git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git + cd arrow/cpp + mkdir release && cd release + ARROW_S3_FLAG="" + if [ "$TARGETARCH" != "s390x" ]; then ARROW_S3_FLAG="-DARROW_S3=ON"; fi + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DARROW_PYTHON=ON \ + -DARROW_PARQUET=ON \ + -DARROW_ORC=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_JSON=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_WITH_LZ4=OFF \ + -DARROW_WITH_ZSTD=OFF \ + -DARROW_WITH_SNAPPY=OFF \ + ${ARROW_S3_FLAG} \ + -DARROW_SUBSTRAIT=ON \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_BENCHMARKS=OFF \ + .. + make -j$(nproc) VERBOSE=1 + make install -j$(nproc) + cd ../../python + pip install --no-cache-dir -r requirements-build.txt + PYARROW_WITH_PARQUET=1 \ + PYARROW_WITH_DATASET=1 \ + PYARROW_WITH_FILESYSTEM=1 \ + PYARROW_WITH_JSON=1 \ + PYARROW_WITH_CSV=1 \ + PYARROW_PARALLEL=$(nproc) \ + python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel + mkdir -p /tmp/wheels + cp dist/pyarrow-*.whl /tmp/wheels/ + chmod -R 777 /tmp/wheels + # Ensure wheels directory exists and has content + ls -la /tmp/wheels/ +else + # Create empty wheels directory for non-s390x + mkdir -p /tmp/wheels +fi +EOF ####################################################### # common-builder (for Power-only) @@ -264,6 +272,7 @@ USER 0 # Dependencies for PDF export begin RUN ./utils/install_pdf_deps.sh ENV PATH="/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:$PATH" + # Dependencies for PDF export end USER 1001 @@ -299,8 +308,12 @@ WORKDIR /opt/app-root/bin USER root # Install useful OS packages -RUN dnf install -y jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat && \ - dnf clean all && rm -rf /var/cache/yum +RUN /bin/bash <<'EOF' +set -Eeuxo pipefail +dnf install -y jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat +dnf clean all +rm -rf /var/cache/yum +EOF # Copy dynamically-linked mongocli built in earlier build stage COPY --from=mongocli-builder /tmp/mongocli /opt/app-root/bin/ @@ -312,11 +325,14 @@ ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ # Copy wheels from build stage (ppc64le and s390x only) COPY --from=pyarrow-builder /tmp/wheels /tmp/wheels -RUN if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \ - pip install --no-cache-dir /tmp/wheels/*.whl; \ -else \ - echo "Skipping wheel install for $TARGETARCH"; \ +RUN /bin/bash <<'EOF' +set -Eeuxo pipefail +if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then + pip install --no-cache-dir /tmp/wheels/*.whl +else + echo "Skipping wheel install for $TARGETARCH" fi +EOF # Copy OpenBLAS,ONNX wheels for Power COPY --from=openblas-builder /root/OpenBLAS-${OPENBLAS_VERSION} /openblas @@ -359,41 +375,43 @@ COPY ${DATASCIENCE_SOURCE_CODE}/pylock.toml ./ # Copy Elyra setup to utils so that it's sourced at startup COPY ${DATASCIENCE_SOURCE_CODE}/setup-elyra.sh ${DATASCIENCE_SOURCE_CODE}/utils ./utils/ -RUN --mount=type=cache,target=/root/.cache/pip \ - echo "Installing software and packages" && \ +RUN --mount=type=cache,target=/root/.cache/pip /bin/bash <<'EOF' +set -Eeuxo pipefail +echo "Installing software and packages" +# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, +# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. +if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then + # We need special flags and environment variables when building packages + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \ + CFLAGS="-O3" CXXFLAGS="-O3" \ + uv pip install --strict --no-deps --no-cache --no-config --no-progress \ + --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \ + --requirements=./pylock.toml +else # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, - # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. - if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \ - # We need special flags and environment variables when building packages - GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \ - CFLAGS="-O3" CXXFLAGS="-O3" \ - uv pip install --strict --no-deps --no-cache --no-config --no-progress \ - --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \ - --requirements=./pylock.toml; \ - else \ - # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, - # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. - uv pip install --strict --no-deps --no-cache --no-config --no-progress \ - --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \ - --requirements=./pylock.toml; \ - fi && \ - # setup path for runtime configuration - mkdir /opt/app-root/runtimes && \ - mkdir /opt/app-root/pipeline-runtimes && \ - # Remove default Elyra runtime-images \ - rm /opt/app-root/share/jupyter/metadata/runtime-images/*.json && \ - # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \ - sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" \ - /opt/app-root/share/jupyter/kernels/python3/kernel.json && \ - # copy jupyter configuration - install -D -m 0644 /opt/app-root/bin/utils/jupyter_server_config.py \ - /opt/app-root/etc/jupyter/jupyter_server_config.py && \ - # Disable announcement plugin of jupyterlab \ - jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \ - # Apply JupyterLab addons \ - /opt/app-root/bin/utils/addons/apply.sh && \ - # Fix permissions to support pip in Openshift environments \ - chmod -R g+w /opt/app-root/lib/python3.12/site-packages && \ - fix-permissions /opt/app-root -P + # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. + uv pip install --strict --no-deps --no-cache --no-config --no-progress \ + --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \ + --requirements=./pylock.toml +fi +# setup path for runtime configuration +mkdir /opt/app-root/runtimes +mkdir /opt/app-root/pipeline-runtimes +# Remove default Elyra runtime-images +rm /opt/app-root/share/jupyter/metadata/runtime-images/*.json +# Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y +sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" \ + /opt/app-root/share/jupyter/kernels/python3/kernel.json +# copy jupyter configuration +install -D -m 0644 /opt/app-root/bin/utils/jupyter_server_config.py \ + /opt/app-root/etc/jupyter/jupyter_server_config.py +# Disable announcement plugin of jupyterlab +jupyter labextension disable "@jupyterlab/apputils-extension:announcements" +# Apply JupyterLab addons +/opt/app-root/bin/utils/addons/apply.sh +# Fix permissions to support pip in Openshift environments +chmod -R g+w /opt/app-root/lib/python3.12/site-packages +fix-permissions /opt/app-root -P +EOF WORKDIR /opt/app-root/src