Merge pull request #45 from deepnote/ach/fix-installation-python

Fix installation python

Author: chudyandrej

.circleci/config.yml

@@ -5,13 +5,6 @@ orbs:
   docker: circleci/docker@2.8
 
 commands:
-  split_python_version:
-    steps:
-      - run:
-          name: Split python version
-          command: |
-            echo "export PYTHON_VERSION=$(echo $PYTHON_VERSION_PATCH | cut -d'.' -f1,2)" >> $BASH_ENV
-            source $BASH_ENV
   setup_buildkit_builder:
     steps:
       - run:
@@ -69,25 +62,23 @@ jobs:
       python-version:
         type: string
     environment:
-      PYTHON_VERSION_PATCH: << parameters.python-version >>
+      PYTHON_VERSION: << parameters.python-version >>
     steps:
       - checkout
       - docker_login
-      - split_python_version # This provides PYTHON_VERSION
       - setup_buildkit_builder
       - run:
           name: Build and push python image
           command: |
             REPOSITORY="deepnote/python"
             TAG="${PYTHON_VERSION}"
             TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+
             docker buildx build \
-              --file=./python/python/Dockerfile.python \
+              --file=./python/python/Dockerfile.python${PYTHON_VERSION} \
               --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST} \
-              --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
-              --build-arg PYTHON_VERSION_PATCH=${PYTHON_VERSION_PATCH} \
               --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
               --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
               --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
@@ -105,11 +96,10 @@ jobs:
       python-version:
         type: string
     environment:
-      PYTHON_VERSION_PATCH: << parameters.python-version >>
+      PYTHON_VERSION: << parameters.python-version >>
     steps:
       - checkout
       - docker_login
-      - split_python_version # This provides PYTHON_VERSION
       - setup_buildkit_builder
       - run:
           name: Build and push python-datascience image
@@ -140,11 +130,10 @@ jobs:
       python-version:
         type: string
     environment:
-      PYTHON_VERSION_PATCH: << parameters.python-version >>
+      PYTHON_VERSION: << parameters.python-version >>
     steps:
       - checkout
       - docker_login
-      - split_python_version # This provides PYTHON_VERSION
       - setup_buildkit_builder
       - run:
           name: Build and push python-conda image
@@ -275,10 +264,10 @@ jobs:
               ./gpu
 
 python-versions: &python-versions
-  - "3.8.19"
-  - "3.9.19"
-  - "3.10.15"
-  - "3.11.10"
+  - "3.8"
+  - "3.9"
+  - "3.10"
+  - "3.11"
 
 r-versions: &r-versions
   - "4.0.4"
@@ -316,7 +305,7 @@ workflows:
             parameters:
               r-version: *r-versions
           requires:
-            - Python 3.9.19
+            - Python 3.9
       - build-and-push-ir-with-libs:
           name: R with libs << matrix.r-version >>
           matrix:

python/base/Dockerfile.base

@@ -1,27 +1,28 @@
-FROM debian:bullseye-20241111-slim
+## Source of the image with list included dependencies!
+# https://github.com/docker-library/buildpack-deps/blob/master/debian/bullseye/Dockerfile
+FROM buildpack-deps:bullseye
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install OS dependencies
-RUN apt-get update && apt-get -yq dist-upgrade \
-&& apt-get install -yq --no-install-recommends \
-build-essential \
-bzip2 \
-cmake \
-curl \
-git \
-graphviz \
-libgtk2.0-dev \
-locales \
-sudo \
-unzip \
-vim \
-wget \
-ssh \
-gnupg2 \
-ca-certificates \
-&& apt-get clean \
-&& rm -rf /var/lib/apt/lists/*
-
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+  build-essential \
+  bzip2 \
+  cmake \
+  curl \
+  git \
+  graphviz \
+  libgtk2.0-dev \
+  locales \
+  sudo \
+  unzip \
+  vim \
+  wget \
+  ssh \
+  gnupg2 \
+  ca-certificates \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
 ENV SHELL=/bin/bash \
   LC_ALL=en_US.UTF-8 \
   LANG=en_US.UTF-8 \

python/python/Dockerfile.python

@@ -0,0 +1,117 @@
+## WARNING: This image is plane copy of: 
+## https://github.com/docker-library/python/blob/7c8595e8e2b1c8bca0b6d9d146675b94c2a37ec7/3.10/bullseye/Dockerfile
+## This is temporary solution and we come with proper solution in the future.
+## Ticket: https://linear.app/deepnote/issue/PLA-3219/cleanup-build-pipeline-for-deepnote-python-images
+
+ARG CIRCLE_PULL_REQUEST
+FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
+
+
+# ensure local python is preferred over distribution python
+ENV PATH /usr/local/bin:$PATH
+
+# cannot remove LANG even though https://bugs.python.org/issue19846 is fixed
+# last attempted removal of LANG broke many users:
+# https://github.com/docker-library/python/pull/570
+ENV LANG C.UTF-8
+
+# runtime dependencies
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		libbluetooth-dev \
+		tk-dev \
+		uuid-dev \
+	; \
+	rm -rf /var/lib/apt/lists/*
+
+ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D
+ENV PYTHON_VERSION 3.10.15
+
+RUN set -eux; \
+	\
+	wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \
+	wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \
+	GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
+	gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \
+	gpg --batch --verify python.tar.xz.asc python.tar.xz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME" python.tar.xz.asc; \
+	mkdir -p /usr/src/python; \
+	tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \
+	rm python.tar.xz; \
+	\
+	cd /usr/src/python; \
+	gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	./configure \
+		--build="$gnuArch" \
+		--enable-loadable-sqlite-extensions \
+		--enable-optimizations \
+		--enable-option-checking=fatal \
+		--enable-shared \
+		--with-lto \
+		--with-system-expat \
+		--with-ensurepip \
+	; \
+	nproc="$(nproc)"; \
+	EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \
+	LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \
+	make -j "$nproc" \
+		"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+		"LDFLAGS=${LDFLAGS:-}" \
+		"PROFILE_TASK=${PROFILE_TASK:-}" \
+	; \
+# https://github.com/docker-library/python/issues/784
+# prevent accidental usage of a system installed libpython of the same version
+	rm python; \
+	make -j "$nproc" \
+		"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+		"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \
+		"PROFILE_TASK=${PROFILE_TASK:-}" \
+		python \
+	; \
+	make install; \
+	\
+# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701
+	bin="$(readlink -ve /usr/local/bin/python3)"; \
+	dir="$(dirname "$bin")"; \
+	mkdir -p "/usr/share/gdb/auto-load/$dir"; \
+	cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \
+	\
+	cd /; \
+	rm -rf /usr/src/python; \
+	\
+	find /usr/local -depth \
+		\( \
+			\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+			-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \
+		\) -exec rm -rf '{}' + \
+	; \
+	\
+	ldconfig; \
+	\
+	export PYTHONDONTWRITEBYTECODE=1; \
+	python3 --version; \
+	\
+	pip3 install \
+		--disable-pip-version-check \
+		--no-cache-dir \
+		--no-compile \
+		'setuptools==65.5.1' \
+		wheel \
+	; \
+	pip3 --version
+
+# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends)
+RUN set -eux; \
+	for src in idle3 pip3 pydoc3 python3 python3-config; do \
+		dst="$(echo "$src" | tr -d 3)"; \
+		[ -s "/usr/local/bin/$src" ]; \
+		[ ! -e "/usr/local/bin/$dst" ]; \
+		ln -svT "$src" "/usr/local/bin/$dst"; \
+	done
+
+
+# We create the virtual environment in the home directory in the Dockerfile
+# for performance improvement.
+RUN python -m venv --system-site-packages ~/venv