Merge pull request #41 from deepnote/ach/build-ir-on-python-3.9

Rebuild IR image with puthon 3.9

Author: hc2p

.circleci/config.yml

@@ -4,7 +4,6 @@ orbs:
   aws-cli: circleci/aws-cli@5.1.1
   docker: circleci/docker@2.8
 
-
 commands:
   split_python_version:
     steps:
@@ -61,7 +60,7 @@ jobs:
               --provenance=false \
               --sbom=false \
               --output type=registry,push=true \
-              .
+              ./python/base
 
   build-and-push-python:
     machine:
@@ -97,7 +96,7 @@ jobs:
               --provenance=false \
               --sbom=false \
               --output type=registry,push=true \
-              .
+              ./python/python
 
   build-and-push-python-datascience:
     machine:
@@ -167,7 +166,72 @@ jobs:
               --provenance=false \
               --sbom=false \
               --output type=registry,push=true \
-              .
+              ./python/conda
+
+  build-and-push-ir:
+    machine:
+      image: ubuntu-2404:current
+    parameters:
+      r-version:
+        type: string
+    environment:
+      R_VERSION: << parameters.r-version >>
+      PYTHON_VERSION: "3.9"
+    steps:
+      - checkout
+      - docker_login
+      - setup_buildkit_builder
+      - run:
+          name: Build and push ir image
+          command: |
+            REPOSITORY="deepnote/ir"
+            TAG="${R_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            docker buildx build \
+              --file=./ir/ir-base/Dockerfile.ir \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG} \
+              --tag=docker.io/${REPOSITORY}:${TAG} \
+              --build-arg R_VERSION=${R_VERSION} \
+              --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --progress plain \
+              --platform linux/amd64 \
+              --provenance=false \
+              --sbom=false \
+              --output type=registry,push=true \
+              ./ir/ir-base
+  build-and-push-ir-with-libs:
+    machine:
+      image: ubuntu-2404:current
+    parameters:
+      r-version:
+        type: string
+    environment:
+      R_VERSION: << parameters.r-version >>
+      PYTHON_VERSION: "3.9"
+    steps:
+      - checkout
+      - docker_login
+      - setup_buildkit_builder
+      - run:
+          name: Build and push ir with datascience libraries
+          command: |
+            REPOSITORY="deepnote/ir-with-libs"
+            TAG="${R_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            docker buildx build \
+              --file=./ir/ir-with-libs/Dockerfile.ir \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG} \
+              --tag=docker.io/${REPOSITORY}:${TAG} \
+              --build-arg R_VERSION=${R_VERSION} \
+              --build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST} \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --progress plain \
+              --platform linux/amd64 \
+              --provenance=false \
+              --sbom=false \
+              --output type=registry,push=true \
+              ./ir/ir-with-libs
 
   build-and-push-gpu:
     machine:
@@ -208,14 +272,18 @@ jobs:
               --provenance=false \
               --sbom=false \
               --output type=registry,push=true \
-              .
+              ./gpu
 
 python-versions: &python-versions
   - "3.8.19"
   - "3.9.19"
   - "3.10.15"
   - "3.11.10"
 
+r-versions: &r-versions
+  - "4.0.4"
+  - "4.2.0"
+
 workflows:
   build:
     jobs:
@@ -242,6 +310,20 @@ workflows:
               python-version: *python-versions
           requires:
             - Building the base image
+      - build-and-push-ir:
+          name: R << matrix.r-version >>
+          matrix:
+            parameters:
+              r-version: *r-versions
+          requires:
+            - Python 3.9.19
+      - build-and-push-ir-with-libs:
+          name: R with libs << matrix.r-version >>
+          matrix:
+            parameters:
+              r-version: *r-versions
+          requires:
+            - R << matrix.r-version >>
 
       - build-and-push-gpu:
           name: Tensorflow 2.9

gpu/Dockerfile

@@ -23,17 +23,17 @@ RUN apt-get update && apt-get -y dist-upgrade \
   gnupg2 \
   && rm -rf /var/lib/apt/lists/*
 
-# replace python version (optional) and 
+# replace python version (optional) and
 # create the virtual environment in the home directory in the Dockerfile
 # for performance improvement.
 ARG PYTHON_VER
 ARG TF_VERSION
-ADD ./gpu/setup.python.sh /tmp/setup.python.sh
+ADD ./setup.python.sh /tmp/setup.python.sh
 RUN bash /tmp/setup.python.sh ${PYTHON_VER} ${TF_VERSION}; rm -v /tmp/setup.python.sh
 
 # setup cuda for tensorflow
 ARG CUDA_IMAGE_TAG
-ADD ./gpu/setup.cuda.sh /tmp/setup.cuda.sh
+ADD ./setup.cuda.sh /tmp/setup.cuda.sh
 RUN bash /tmp/setup.cuda.sh ${CUDA_IMAGE_TAG}; rm -v /tmp/setup.cuda.sh
 
 ENV DEEPNOTE_PYTHON_KERNEL_ONLY=true

ir-with-libs/Dockerfile

@@ -0,0 +1,35 @@
+# Define build arguments for Python and R versions
+ARG PYTHON_VERSION=3.9
+# Use the specified Python version as the base image
+FROM deepnote/python:${PYTHON_VERSION}
+
+# Set environment variables
+ARG R_VERSION=4.4.0
+# Use the R version specified by the build argument
+ENV R_VERSION="${R_VERSION}" \
+	R_HOME="/usr/local/lib/R" \
+	TZ="Etc/UTC" \
+	DEFAULT_KERNEL_NAME="ir" \
+	DEEPNOTE_PYTHON_KERNEL_ONLY=""
+
+# Copy the R installation script and run it
+COPY install_R_source.sh /rocker_scripts/install_R_source.sh
+RUN chmod +x /rocker_scripts/install_R_source.sh && \
+	/rocker_scripts/install_R_source.sh
+
+# Set the working directory
+WORKDIR /opt
+
+# Create and activate a virtual environment in one RUN command
+RUN python -m venv jupyter-env && \
+	. ./jupyter-env/bin/activate && \
+	pip install --upgrade pip notebook
+
+# Install IRkernel and configure it
+RUN . ./jupyter-env/bin/activate && \
+	R -e "install.packages('IRkernel', repos='http://cran.rstudio.com/')" && \
+	R -e "IRkernel::installspec()"
+
+# Clean up unnecessary files to reduce the image size
+RUN apt-get clean && \
+	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*