Skip to content

Commit b2767d8

Browse files
oestebanoesteban
authored
Merge pull request #1990 from mgxd/maint/docker-registry
MAINT: Use local docker registry
2 parents 1641529 + 0a2c9a1 commit b2767d8

File tree

2 files changed

+143
-110
lines changed

2 files changed

+143
-110
lines changed

‎.circleci/config.yml‎

Lines changed: 142 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ jobs:
55
environment:
66
TZ: "/usr/share/zoneinfo/America/Los_Angeles"
77
SCRATCH: "/scratch"
8-
docker:
9-
- image: docker:18.01.0-ce-git
8+
machine:
9+
image: circleci/classic:201711-01
1010
working_directory: /tmp/src/fmriprep
1111
steps:
1212
- checkout
@@ -18,33 +18,46 @@ jobs:
1818
circleci step halt
1919
fi
2020
21-
- run:
22-
name: Install parallel gzip and python3
23-
command: |
24-
apk add --no-cache pigz python3
2521
- restore_cache:
2622
keys:
27-
- docker-v3-{{ .Branch }}-{{ .Revision }}
28-
- docker-v3-{{ .Branch }}-
29-
- docker-v3-master-
30-
- docker-v3-
23+
- build-v4-{{ .Branch }}-{{ epoch }}
24+
- build-v4-{{ .Branch }}-
25+
- build-v4-master-
26+
- build-v4-
3127
paths:
32-
- /tmp/cache/docker.tar.gz
33-
- setup_remote_docker
28+
- /tmp/docker
3429
- run:
35-
name: Load Docker image layer cache
36-
no_output_timeout: 30m
30+
name: Set up Docker registry
3731
command: |
38-
docker info
39-
set +o pipefail
40-
if [ -f /tmp/cache/docker.tar.gz ]; then
41-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
42-
docker images
32+
docker run -d -p 5000:5000 --restart=always --name=registry \
33+
-v /tmp/docker:/var/lib/registry registry:2
34+
- run:
35+
name: Pull images
36+
command: |
37+
set +e
38+
docker pull localhost:5000/ubuntu
39+
success=$?
40+
set -e
41+
if [[ "$success" = "0" ]]; then
42+
echo "Pulling from local registry"
43+
docker tag localhost:5000/ubuntu ubuntu:xenial-20200114
44+
docker pull localhost:5000/fmriprep
45+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
46+
docker tag localhost:5000/fmriprep poldracklab/fmriprep
47+
else
48+
echo "Pulling from Docker Hub"
49+
docker pull ubuntu:xenial-20200114
50+
docker tag ubuntu:xenial-20200114 localhost:5000/ubuntu
51+
docker push localhost:5000/ubuntu
52+
docker pull poldracklab/fmriprep:latest
4353
fi
4454
- run:
4555
name: Build Docker image
4656
no_output_timeout: 60m
4757
command: |
58+
export PY3=$(pyenv versions | grep '3\.' |
59+
sed -e 's/.* 3\./3./' -e 's/ .*//')
60+
pyenv local $PY3
4861
# Get version, update files.
4962
THISVERSION=$( python3 get_version.py )
5063
if [[ ${THISVERSION:0:1} == "0" ]] ; then
@@ -58,52 +71,34 @@ jobs:
5871
sed -i "s/title = {FMRIPrep}/title = {FMRIPrep ${CIRCLE_TAG:-$THISVERSION}}/" fmriprep/data/boilerplate.bib
5972
# Build docker image
6073
e=1 && for i in {1..5}; do
61-
docker build \
74+
docker build --rm \
6275
--cache-from=poldracklab/fmriprep \
63-
--rm=false \
6476
-t poldracklab/fmriprep:latest \
6577
--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
6678
--build-arg VCS_REF=`git rev-parse --short HEAD` \
6779
--build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \
6880
&& e=0 && break || sleep 15
6981
done && [ "$e" -eq "0" ]
70-
# test if development image should be built
71-
if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[build[ _]?devel\]' )" != "" ]]; then
72-
echo "building development container"
73-
docker tag poldracklab/fmriprep poldracklab/fmriprep:unstable
74-
docker build \
75-
--rm=false \
76-
-t poldracklab/fmriprep_devel \
77-
-f Dockerfile_devel .
78-
fi
7982
- run:
80-
name: Smoke test Docker image
83+
name: Docker push to local registry
84+
no_output_timeout: 40m
8185
command: |
82-
THISVERSION=$( python3 get_version.py )
83-
THISVERSION=${THISVERSION%.dirty*}
84-
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
85-
DOCKERVERSION=$(docker run --rm -it poldracklab/fmriprep:latest --version)
86-
DOCKERVERSION=${DOCKERVERSION%$'\r'}
87-
DOCKERVERSION=${DOCKERVERSION#*"fMRIPrep v"}
88-
echo "VERSION: \"$THISVERSION\""
89-
echo "DOCKERVERSION: \"${DOCKERVERSION}\""
90-
test "$DOCKERVERSION" = "$THISVERSION"
86+
docker tag poldracklab/fmriprep:latest localhost:5000/fmriprep
87+
docker push localhost:5000/fmriprep
9188
- run:
92-
name: Docker save
93-
no_output_timeout: 40m
89+
name: Docker registry garbage collection
9490
command: |
95-
mkdir -p /tmp/cache
96-
docker save ubuntu:xenial-20161213 poldracklab/fmriprep:latest \
97-
| pigz -3 > /tmp/cache/docker.tar.gz
91+
docker exec -it registry /bin/registry garbage-collect --delete-untagged \
92+
/etc/docker/registry/config.yml
9893
9994
- persist_to_workspace:
10095
root: /tmp
10196
paths:
10297
- src/fmriprep
10398
- save_cache:
104-
key: docker-v3-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
99+
key: build-v4-{{ .Branch }}-{{ epoch }}
105100
paths:
106-
- /tmp/cache/docker.tar.gz
101+
- /tmp/docker
107102

108103
get_data:
109104
machine:
@@ -210,18 +205,24 @@ jobs:
210205
at: /tmp
211206
- restore_cache:
212207
keys:
213-
- docker-v3-{{ .Branch }}-{{ .Revision }}
208+
- build-v4-{{ .Branch }}-{{ epoch }}
209+
- build-v4-{{ .Branch }}-
210+
- build-v4-
211+
paths:
212+
- /tmp/docker
213+
- restore_cache:
214+
keys:
215+
- data-v8-{{ .Revision }}
214216
- run:
215-
name: Load Docker image layer cache
216-
no_output_timeout: 30m
217+
name: Set-up a Docker registry
217218
command: |
218-
docker info
219-
set +o pipefail
220-
if [ -f /tmp/cache/docker.tar.gz ]; then
221-
sudo apt update && sudo apt -y install pigz
222-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
223-
docker images
224-
fi
219+
docker run -d -p 5000:5000 --restart=always --name=registry \
220+
-v /tmp/docker:/var/lib/registry registry:2
221+
- run:
222+
name: Pull images from local registry
223+
command: |
224+
docker pull localhost:5000/fmriprep
225+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
225226
- run:
226227
name: Run fMRIPrep tests
227228
no_output_timeout: 2h
@@ -323,7 +324,11 @@ jobs:
323324
at: /tmp
324325
- restore_cache:
325326
keys:
326-
- docker-v3-{{ .Branch }}-{{ .Revision }}
327+
- build-v4-{{ .Branch }}-{{ epoch }}
328+
- build-v4-{{ .Branch }}-
329+
- build-v4-
330+
paths:
331+
- /tmp/docker
327332
- restore_cache:
328333
keys:
329334
- data-v7-{{ .Branch }}-{{ .Revision }}
@@ -333,23 +338,22 @@ jobs:
333338
- ds005-anat-v17-{{ .Branch }}
334339
- ds005-anat-v17-master
335340
- ds005-anat-v17-
341+
- run:
342+
name: Set-up a Docker registry
343+
command: |
344+
docker run -d -p 5000:5000 --restart=always --name=registry \
345+
-v /tmp/docker:/var/lib/registry registry:2
346+
- run:
347+
name: Pull images from local registry
348+
command: |
349+
docker pull localhost:5000/fmriprep
350+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
336351
- run:
337352
name: Setting up test
338353
command: |
339354
mkdir -p /tmp/ds005/derivatives && sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
340355
pip install --upgrade pip setuptools
341356
pip install --upgrade /tmp/src/fmriprep/wrapper/
342-
- run:
343-
name: Load Docker image layer cache
344-
no_output_timeout: 30m
345-
command: |
346-
docker info
347-
set +o pipefail
348-
if [ -f /tmp/cache/docker.tar.gz ]; then
349-
sudo apt update && sudo apt -y install pigz
350-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
351-
docker images
352-
fi
353357
- run:
354358
name: Run anatomical workflow on ds005
355359
no_output_timeout: 2h
@@ -479,7 +483,11 @@ jobs:
479483
at: /tmp
480484
- restore_cache:
481485
keys:
482-
- docker-v3-{{ .Branch }}-{{ .Revision }}
486+
- build-v4-{{ .Branch }}-{{ epoch }}
487+
- build-v4-{{ .Branch }}-
488+
- build-v4-
489+
paths:
490+
- /tmp/docker
483491
- restore_cache:
484492
keys:
485493
- data-v7-{{ .Branch }}-{{ .Revision }}
@@ -496,16 +504,15 @@ jobs:
496504
pip install --upgrade pip setuptools
497505
pip install --upgrade /tmp/src/fmriprep/wrapper/
498506
- run:
499-
name: Load Docker image layer cache
500-
no_output_timeout: 30m
507+
name: Set-up a Docker registry
501508
command: |
502-
docker info
503-
set +o pipefail
504-
if [ -f /tmp/cache/docker.tar.gz ]; then
505-
sudo apt update && sudo apt -y install pigz
506-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
507-
docker images
508-
fi
509+
docker run -d -p 5000:5000 --restart=always --name=registry \
510+
-v /tmp/docker:/var/lib/registry registry:2
511+
- run:
512+
name: Pull images from local registry
513+
command: |
514+
docker pull localhost:5000/fmriprep
515+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
509516
- run:
510517
name: Run anatomical workflow on ds054
511518
no_output_timeout: 2h
@@ -617,7 +624,11 @@ jobs:
617624
at: /tmp
618625
- restore_cache:
619626
keys:
620-
- docker-v3-{{ .Branch }}-{{ .Revision }}
627+
- build-v4-{{ .Branch }}-{{ epoch }}
628+
- build-v4-{{ .Branch }}-
629+
- build-v4-
630+
paths:
631+
- /tmp/docker
621632
- restore_cache:
622633
keys:
623634
- data-v7-{{ .Branch }}-{{ .Revision }}
@@ -634,16 +645,15 @@ jobs:
634645
pip install --upgrade pip setuptools
635646
pip install --upgrade /tmp/src/fmriprep/wrapper/
636647
- run:
637-
name: Load Docker image layer cache
638-
no_output_timeout: 30m
648+
name: Set-up a Docker registry
639649
command: |
640-
docker info
641-
set +o pipefail
642-
if [ -f /tmp/cache/docker.tar.gz ]; then
643-
sudo apt update && sudo apt -y install pigz
644-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
645-
docker images
646-
fi
650+
docker run -d -p 5000:5000 --restart=always --name=registry \
651+
-v /tmp/docker:/var/lib/registry registry:2
652+
- run:
653+
name: Pull images from local registry
654+
command: |
655+
docker pull localhost:5000/fmriprep
656+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
647657
- run:
648658
name: Run anatomical workflow on ds000210
649659
no_output_timeout: 2h
@@ -712,20 +722,40 @@ jobs:
712722
working_directory: /tmp/src/fmriprep
713723
steps:
714724

725+
- run:
726+
name: Check whether image should be deployed to Docker Hub
727+
command: |
728+
if [[ "$CIRCLE_PROJECT_USERNAME" != "poldracklab" ]]; then
729+
echo "Nothing to deploy for $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME."
730+
circleci step halt
731+
fi
732+
- checkout:
733+
path: /home/circleci/src/fmriprep
734+
- run:
735+
name: Check whether build should be skipped
736+
command: |
737+
cd /home/circleci/src/fmriprep
738+
if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
739+
echo "Only docs build"
740+
circleci step halt
741+
fi
715742
- restore_cache:
716743
keys:
717-
- docker-v3-{{ .Branch }}-{{ .Revision }}
744+
- build-v4-{{ .Branch }}-{{ epoch }}
745+
- build-v4-{{ .Branch }}-
746+
- build-v4-
747+
paths:
748+
- /tmp/docker
718749
- run:
719-
name: Load Docker image layer cache
720-
no_output_timeout: 30m
750+
name: Setup a Docker registry
721751
command: |
722-
docker info
723-
set +o pipefail
724-
if [ -f /tmp/cache/docker.tar.gz ]; then
725-
sudo apt update && sudo apt -y install pigz
726-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
727-
docker images
728-
fi
752+
docker run -d -p 5000:5000 --restart=always --name=registry \
753+
-v /tmp/docker:/var/lib/registry registry:2
754+
- run:
755+
name: Pull images from local registry
756+
command: |
757+
docker pull localhost:5000/fmriprep
758+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
729759
- run:
730760
name: Deploy to Docker Hub
731761
no_output_timeout: 40m
@@ -753,18 +783,21 @@ jobs:
753783
fi
754784
- restore_cache:
755785
keys:
756-
- docker-v3-{{ .Branch }}-{{ .Revision }}
786+
- build-v4-{{ .Branch }}-{{ epoch }}
787+
- build-v4-{{ .Branch }}-
788+
- build-v4-
789+
paths:
790+
- /tmp/docker
757791
- run:
758-
name: Load Docker image layer cache
759-
no_output_timeout: 30m
792+
name: Set-up a Docker registry
760793
command: |
761-
docker info
762-
set +o pipefail
763-
if [ -f /tmp/cache/docker.tar.gz ]; then
764-
sudo apt update && sudo apt -y install pigz
765-
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
766-
docker images
767-
fi
794+
docker run -d -p 5000:5000 --restart=always --name=registry \
795+
-v /tmp/docker:/var/lib/registry registry:2
796+
- run:
797+
name: Pull images from local registry
798+
command: |
799+
docker pull localhost:5000/fmriprep
800+
docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
768801
- run:
769802
name: Deploy to Docker Hub
770803
no_output_timeout: 40m

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Use Ubuntu 16.04 LTS
2-
FROM ubuntu:xenial-20161213
2+
FROM ubuntu:xenial-20200114
33

44
# Pre-cache neurodebian key
55
COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg

0 commit comments

Comments
 (0)