Skip to content

Commit df2f021

Browse files
hc2phc2p
authored
Merge pull request #39 from deepnote/hannes/pla-3118-improve-layer-reuse-of-python-images
Improve layer reuse of python images
2 parents 656b1c6 + 12228e4 commit df2f021

File tree

6 files changed

+176
-115
lines changed

6 files changed

+176
-115
lines changed

‎.circleci/config.yml‎

Lines changed: 87 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,74 @@
11
version: 2.1
22

33
orbs:
4-
docker: circleci/docker@2.2.0
4+
docker: circleci/docker@2.8
55

6+
7+
commands:
8+
split_python_version:
9+
steps:
10+
- run:
11+
name: Split python version
12+
command: |
13+
echo "export PYTHON_VERSION=$(echo $PYTHON_VERSION_PATCH | cut -d'.' -f1,2)" >> $BASH_ENV
14+
source $BASH_ENV
15+
setup_buildkit_builder:
16+
steps:
17+
- run:
18+
name: Setup BuildKit builder
19+
command: |
20+
docker buildx create \
21+
--name custom-builder \
22+
--use \
23+
--driver docker-container \
624
jobs:
25+
build-and-push-base:
26+
executor: docker/docker
27+
steps:
28+
- checkout
29+
- setup_remote_docker
30+
- setup_buildkit_builder
31+
- docker/check:
32+
use-docker-credentials-store: true
33+
- docker/build:
34+
step-name: Base python image (debian + apt-get deps)
35+
path: ./python/base
36+
dockerfile: Dockerfile.base
37+
extra_build_args: |
38+
--progress plain
39+
--platform linux/amd64
40+
--cache-from type=registry,ref=deepnote/python:base-buildcache,mode=max
41+
--cache-to type=registry,ref=deepnote/python:base-buildcache,mode=max
42+
--output type=registry,push=true
43+
image: deepnote/python
44+
tag: base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
745
build-and-push-python:
846
executor: docker/docker
947
parameters:
1048
python-version:
1149
type: string
1250
environment:
13-
PYTHON_VERSION: << parameters.python-version >>
51+
PYTHON_VERSION_PATCH: << parameters.python-version >>
1452
steps:
1553
- checkout
1654
- setup_remote_docker
1755
- docker/check:
1856
use-docker-credentials-store: true
57+
- split_python_version
58+
- setup_buildkit_builder
1959
- docker/build:
20-
step-name: Base Python image
60+
step-name: Python image
2161
path: ./python/base
22-
dockerfile: Dockerfile
23-
extra_build_args: "--build-arg FROM_PYTHON_VERSION=${PYTHON_VERSION}"
24-
image: deepnote/python
25-
tag: ${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
26-
- docker/push:
62+
dockerfile: Dockerfile.python
63+
extra_build_args: |
64+
--build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}
65+
--build-arg PYTHON_VERSION=${PYTHON_VERSION}
66+
--build-arg PYTHON_VERSION_PATCH=${PYTHON_VERSION_PATCH}
67+
--progress plain
68+
--platform linux/amd64
69+
--cache-from type=registry,ref=deepnote/python:${PYTHON_VERSION}-buildcache,mode=max
70+
--cache-to type=registry,ref=deepnote/python:${PYTHON_VERSION}-buildcache,mode=max
71+
--output type=registry,push=true
2772
image: deepnote/python
2873
tag: ${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
2974
build-and-push-python-datascience:
@@ -32,24 +77,30 @@ jobs:
3277
python-version:
3378
type: string
3479
environment:
35-
PYTHON_VERSION: << parameters.python-version >>
80+
PYTHON_VERSION_PATCH: << parameters.python-version >>
3681
steps:
3782
- checkout
3883
- setup_remote_docker
3984
- docker/check:
4085
use-docker-credentials-store: true
86+
- split_python_version
87+
- setup_buildkit_builder
4188
- docker/build:
4289
step-name: Datascience Python image
4390
path: ./python/datascience
4491
docker-context: ./python/datascience
4592
dockerfile: Dockerfile.datascience
46-
extra_build_args: "--build-arg FROM_PYTHON_VERSION=${PYTHON_VERSION}"
93+
extra_build_args: |
94+
--build-arg PYTHON_VERSION=${PYTHON_VERSION}
95+
--build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}
96+
--progress plain
97+
--platform linux/amd64
98+
--cache-from type=registry,ref=deepnote/python:${PYTHON_VERSION}-buildcache,mode=max
99+
--cache-from type=registry,ref=deepnote/python-datascience:${PYTHON_VERSION}-buildcache,mode=max
100+
--cache-to type=registry,ref=deepnote/python-datascience:${PYTHON_VERSION}-buildcache,mode=max
101+
--output type=registry,push=true
47102
image: deepnote/python-datascience
48103
tag: ${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
49-
- docker/push:
50-
image: deepnote/python-datascience
51-
tag: ${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
52-
53104
build-and-push-gpu:
54105
executor: docker/docker
55106
parameters:
@@ -68,44 +119,42 @@ jobs:
68119
- setup_remote_docker
69120
- docker/check:
70121
use-docker-credentials-store: true
122+
- split_python_version
71123
- docker/build:
72124
step-name: Tensorflow GPU image
73125
path: ./gpu
74-
extra_build_args: "--build-arg CUDA_IMAGE_TAG=${CUDA_TAG} --build-arg TF_VERSION=${TF_VERSION} --build-arg PYTHON_VER=${PY_VERSION}"
126+
extra_build_args: "--build-arg CUDA_IMAGE_TAG=${CUDA_TAG} --build-arg TF_VERSION=${TF_VERSION} --build-arg PYTHON_VER=${PY_VERSION} --progress plain"
75127
image: deepnote/tensorflow
76128
tag: ${TF_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-gpu
77129
- docker/push:
78130
image: deepnote/tensorflow
79131
tag: ${TF_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-gpu
80132

133+
python-versions: &python-versions
134+
- "3.8.19"
135+
- "3.9.19"
136+
- "3.10.15"
137+
- "3.11.10"
138+
81139
workflows:
82140
build:
83141
jobs:
142+
- build-and-push-base:
143+
name: Building the base image
84144
- build-and-push-python:
85-
name: Python 3.8
86-
python-version: "3.8"
87-
- build-and-push-python:
88-
name: Python 3.9
89-
python-version: "3.9"
90-
- build-and-push-python:
91-
name: Python 3.10
92-
python-version: "3.10"
93-
- build-and-push-python:
94-
name: Python 3.11
95-
python-version: "3.11"
96-
97-
- build-and-push-python-datascience:
98-
name: Python Datascience 3.8
99-
python-version: "3.8"
145+
name: Python << matrix.python-version >>
146+
matrix:
147+
parameters:
148+
python-version: *python-versions
149+
requires:
150+
- Building the base image
100151
- build-and-push-python-datascience:
101-
name: Python Datascience 3.9
102-
python-version: "3.9"
103-
- build-and-push-python-datascience:
104-
name: Python Datascience 3.10
105-
python-version: "3.10"
106-
- build-and-push-python-datascience:
107-
name: Python Datascience 3.11
108-
python-version: "3.11"
152+
name: Python Datascience << matrix.python-version >>
153+
matrix:
154+
parameters:
155+
python-version: *python-versions
156+
requires:
157+
- Python << matrix.python-version >>
109158

110159
- build-and-push-gpu:
111160
name: Tensorflow 2.9

‎python/Dockerfile‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎python/base/Dockerfile‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎python/base/Dockerfile.base‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
FROM debian:bullseye-slim
2+
ENV DEBIAN_FRONTEND noninteractive
3+
4+
# Install OS dependencies
5+
RUN apt-get update && apt-get -yq dist-upgrade \
6+
&& apt-get install -yq --no-install-recommends \
7+
build-essential \
8+
bzip2 \
9+
cmake \
10+
curl \
11+
git \
12+
graphviz \
13+
libgtk2.0-dev \
14+
locales \
15+
sudo \
16+
unzip \
17+
vim \
18+
wget \
19+
ssh \
20+
gnupg2 \
21+
ca-certificates \
22+
&& apt-get clean \
23+
&& rm -rf /var/lib/apt/lists/*
24+
25+
ENV SHELL=/bin/bash \
26+
LC_ALL=en_US.UTF-8 \
27+
LANG=en_US.UTF-8 \
28+
LANGUAGE=en_US.UTF-8 \
29+
DEEPNOTE_PYTHON_KERNEL_ONLY=true
30+
31+
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
32+
locale-gen

‎python/base/Dockerfile.python‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
ARG CIRCLE_PULL_REQUEST
2+
FROM debian:bullseye-slim AS builder
3+
4+
# Install dependencies for building Python
5+
RUN apt-get update && apt-get install -y --no-install-recommends \
6+
wget \
7+
cmake \
8+
build-essential \
9+
libssl-dev \
10+
zlib1g-dev \
11+
libncurses5-dev \
12+
libncursesw5-dev \
13+
libreadline-dev \
14+
libsqlite3-dev \
15+
libgdbm-dev \
16+
libdb5.3-dev \
17+
libbz2-dev \
18+
libexpat1-dev \
19+
liblzma-dev \
20+
tk-dev \
21+
libffi-dev \
22+
uuid-dev \
23+
ca-certificates \
24+
&& apt-get clean \
25+
&& rm -rf /var/lib/apt/lists/*
26+
27+
# Download and extract the Python source code
28+
WORKDIR /tmp
29+
ARG PYTHON_VERSION_PATCH
30+
RUN wget -qO- "https://www.python.org/ftp/python/${PYTHON_VERSION_PATCH}/Python-${PYTHON_VERSION_PATCH}.tgz" | tar xvz \
31+
&& cd "/tmp/Python-${PYTHON_VERSION_PATCH}" \
32+
&& ./configure --enable-optimizations --with-ensurepip=install \
33+
&& make -j "$(nproc)" \
34+
&& make altinstall
35+
36+
37+
FROM deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
38+
# Copy Python from the builder stage
39+
ARG PYTHON_VERSION
40+
# Layers will be different between python versions from here onwards because of the build-arg
41+
42+
COPY --from=builder "/usr/local/bin/python${PYTHON_VERSION}" "/usr/local/bin/python${PYTHON_VERSION}"
43+
COPY --from=builder "/usr/local/bin/pip${PYTHON_VERSION}" "/usr/local/bin/pip${PYTHON_VERSION}"
44+
COPY --from=builder "/usr/local/lib/python${PYTHON_VERSION}" "/usr/local/lib/python${PYTHON_VERSION}"
45+
46+
RUN update-alternatives --install /usr/bin/python python "/usr/local/bin/python${PYTHON_VERSION}" 1
47+
RUN update-alternatives --install /usr/bin/pip pip "/usr/local/bin/pip${PYTHON_VERSION}" 1
48+
49+
# We create the virtual environment in the home directory in the Dockerfile
50+
# for performance improvement.
51+
RUN python -m venv --system-site-packages ~/venv

‎python/datascience/Dockerfile.datascience‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
ARG FROM_PYTHON_VERSION=3.8
2-
FROM deepnote/python:${FROM_PYTHON_VERSION}
1+
ARG PYTHON_VERSION=3.8
2+
ARG CIRCLE_PULL_REQUEST
3+
FROM deepnote/python:${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}
34

45
RUN pip install --upgrade pip
56

@@ -10,11 +11,11 @@ ADD requirements-below-3.11.txt /requirements-below-3.11.txt
1011
ADD constraints.txt /constraints.txt
1112

1213
#Determine the Python version and set the version-specifications file
13-
ARG FROM_PYTHON_VERSION
14+
ARG PYTHON_VERSION
1415

1516
RUN python -m pip install --upgrade setuptools pip
1617

17-
RUN if [ "$(printf '%s\n' "$FROM_PYTHON_VERSION" "3.11" | sort -V | head -n1)" = "3.11" ]; then \
18+
RUN if [ "$(printf '%s\n' "$PYTHON_VERSION" "3.11" | sort -V | head -n1)" = "3.11" ]; then \
1819
mv "requirements-3.11+.txt" "requirements.txt" \
1920
; else \
2021
mv "requirements-below-3.11.txt" "requirements.txt" \
@@ -28,4 +29,4 @@ RUN apt-get update \
2829
&& apt-get purge -y gcc \
2930
&& apt-get autoremove -y \
3031
&& apt-get clean \
31-
&& rm -rf /var/lib/apt/lists/* /root/.cache
32+
&& rm -rf /var/lib/apt/lists/* /root/.cache

0 commit comments

Comments
 (0)