From 1c16a3007b132ab93dfe2de0d70a3371108ffafb Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 16 Jun 2022 12:19:25 -0700 Subject: [PATCH 01/15] Use mamba instead of conda for dockerfile package installation --- docker/main.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/main.dockerfile b/docker/main.dockerfile index cae7fb96e..2e0a520fe 100644 --- a/docker/main.dockerfile +++ b/docker/main.dockerfile @@ -6,7 +6,7 @@ LABEL author "Nils Braun " # Install dependencies for dask-sql COPY docker/conda.txt /opt/dask_sql/ RUN conda config --add channels conda-forge \ - && /opt/conda/bin/conda install --freeze-installed \ + && /opt/conda/bin/mamba install --freeze-installed \ "jpype1>=1.0.2" \ "openjdk>=11" \ "maven>=3.6.0" \ @@ -18,7 +18,7 @@ RUN conda config --add channels conda-forge \ "pygments>=2.7.1" \ "dask-ml>=2022.1.22" \ "scikit-learn>=1.0.0" \ - "intake>=0.6.0" \ + "intake>=0.6.0" -y \ && conda clean -ay # install dask-sql From e657bbf9733d29cab7b17670a96b4549d5cc89c2 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 16 Jun 2022 12:47:02 -0700 Subject: [PATCH 02/15] Test docker builds on pr if those files are modified --- .github/workflows/docker.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7f52411cc..692569789 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,9 @@ on: push: branches: - main + pull_request: + paths: + - "docker/**" # When this workflow is queued, automatically cancel any previous running # or pending jobs from the same branch @@ -44,7 +47,7 @@ jobs: platforms: linux/amd64,linux/arm64,linux/386 tags: ${{ steps.docker_meta_main.outputs.tags }} labels: ${{ steps.docker_meta_main.outputs.labels }} - push: true + push: ${{ github.ref == 'refs/head/main' && github.repository == 'dask-contrib/dask-sql' }} - name: Docker meta for cloud image id: docker_meta_cloud uses: crazy-max/ghaction-docker-meta@v1 @@ -59,4 +62,4 @@ jobs: platforms: linux/amd64,linux/arm64,linux/386 tags: ${{ steps.docker_meta_cloud.outputs.tags }} labels: ${{ steps.docker_meta_cloud.outputs.labels }} - push: true + push: ${{ github.ref == 'refs/head/main' && github.repository == 'dask-contrib/dask-sql' }} From 350fb504af7e2a2a326f64fbb1493e2e09710e83 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 16 Jun 2022 12:47:23 -0700 Subject: [PATCH 03/15] Use mamba on cloud dockerfile --- docker/cloud.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/cloud.dockerfile b/docker/cloud.dockerfile index c63406504..d30396c95 100644 --- a/docker/cloud.dockerfile +++ b/docker/cloud.dockerfile @@ -2,9 +2,9 @@ ARG DOCKER_META_VERSION FROM nbraun/dask-sql:${DOCKER_META_VERSION} RUN conda config --add channels conda-forge \ - && /opt/conda/bin/conda install --freeze-installed \ + && /opt/conda/bin/mamba install --freeze-installed \ "s3fs" \ - "dask-cloudprovider" \ + "dask-cloudprovider" -y \ && pip install awscli \ && conda clean -ay From dd12f762d2b5f82e3a238eecd091cededce7ef8c Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 16 Jun 2022 13:49:29 -0700 Subject: [PATCH 04/15] docker login only on pushes to main --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 692569789..ee7403c7a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,6 +29,10 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub + if: | + github.event_name == 'push' + && github.ref == 'refs/heads/main' + && github.repository == 'dask-contrib/dask-sql' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} From e655c35b6ce71faca709d825476ef276be9302bd Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 16 Jun 2022 15:40:01 -0700 Subject: [PATCH 05/15] Load the main image into docker for use by the cloud image --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ee7403c7a..8ebf6c32a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,6 +52,7 @@ jobs: tags: ${{ steps.docker_meta_main.outputs.tags }} labels: ${{ steps.docker_meta_main.outputs.labels }} push: ${{ github.ref == 'refs/head/main' && github.repository == 'dask-contrib/dask-sql' }} + load: true - name: Docker meta for cloud image id: docker_meta_cloud uses: crazy-max/ghaction-docker-meta@v1 From f5e5dd9de93b55d5eb5baafee43417c6b2ff5879 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 15:11:07 -0700 Subject: [PATCH 06/15] Use env for docker_push action Co-authored-by: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> --- .github/workflows/docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8ebf6c32a..8d2dc035a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,7 +20,8 @@ jobs: push_to_registry: name: Push Docker image to Docker Hub runs-on: ubuntu-latest - if: github.repository == 'dask-contrib/dask-sql' + env: + DOCKER_PUSH: ${{ github.event_name == 'push' && github.repository == 'dask-contrib/dask-sql' }} steps: - name: Check out the repo uses: actions/checkout@v2 From ee473815ff8a23ecfdf03e00ec1f58c54be810c5 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 15:13:52 -0700 Subject: [PATCH 07/15] Apply suggestions from code review Co-authored-by: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> --- .github/workflows/docker.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8d2dc035a..9d002673a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,10 +30,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - if: | - github.event_name == 'push' - && github.ref == 'refs/heads/main' - && github.repository == 'dask-contrib/dask-sql' + if: ${{ env.DOCKER_PUSH }} uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} @@ -49,11 +46,11 @@ jobs: context: . file: ./docker/main.dockerfile build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }} - platforms: linux/amd64,linux/arm64,linux/386 + platforms: ${{ env.DOCKER_PUSH && 'linux/amd64,linux/arm64,linux/386' || 'linux/amd64' }} tags: ${{ steps.docker_meta_main.outputs.tags }} labels: ${{ steps.docker_meta_main.outputs.labels }} - push: ${{ github.ref == 'refs/head/main' && github.repository == 'dask-contrib/dask-sql' }} - load: true + push: ${{ env.DOCKER_PUSH }} + load: ${{ !env.DOCKER_PUSH }} - name: Docker meta for cloud image id: docker_meta_cloud uses: crazy-max/ghaction-docker-meta@v1 @@ -65,7 +62,7 @@ jobs: context: . file: ./docker/cloud.dockerfile build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }} - platforms: linux/amd64,linux/arm64,linux/386 + platforms: ${{ env.DOCKER_PUSH && 'linux/amd64,linux/arm64,linux/386' || 'linux/amd64' }} tags: ${{ steps.docker_meta_cloud.outputs.tags }} labels: ${{ steps.docker_meta_cloud.outputs.labels }} - push: ${{ github.ref == 'refs/head/main' && github.repository == 'dask-contrib/dask-sql' }} + push: ${{ env.DOCKER_PUSH }} From 52abd469448182f7a9368002240140410931bdc3 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 15:28:58 -0700 Subject: [PATCH 08/15] Debug env variable value --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9d002673a..5e8120ee0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,6 +22,8 @@ jobs: runs-on: ubuntu-latest env: DOCKER_PUSH: ${{ github.event_name == 'push' && github.repository == 'dask-contrib/dask-sql' }} + run: | + echo ${{ env.DOCKER_PUSH }} steps: - name: Check out the repo uses: actions/checkout@v2 From c0d160c6f28c4563d7024e164df9a4b30fa5a749 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 15:32:17 -0700 Subject: [PATCH 09/15] Debug env variable value --- .github/workflows/docker.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5e8120ee0..8a02f6bf5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,9 +22,10 @@ jobs: runs-on: ubuntu-latest env: DOCKER_PUSH: ${{ github.event_name == 'push' && github.repository == 'dask-contrib/dask-sql' }} - run: | - echo ${{ env.DOCKER_PUSH }} steps: + - name: Test env value + run: | + echo ${{ env.DOCKER_PUSH }} - name: Check out the repo uses: actions/checkout@v2 - name: Set up QEMU From 34b21d41a627ff477999c3e4ce24f6adf17dad7b Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 15:47:54 -0700 Subject: [PATCH 10/15] Use fromJSON to evaluate string env variables as boolean --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8a02f6bf5..161c84dc3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Test env value run: | - echo ${{ env.DOCKER_PUSH }} + echo ${{ fromJSON(env.DOCKER_PUSH) }} - name: Check out the repo uses: actions/checkout@v2 - name: Set up QEMU @@ -33,7 +33,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - if: ${{ env.DOCKER_PUSH }} + if: ${{ fromJSON(env.DOCKER_PUSH) }} uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} From 76a4afbe46c5d6bd0b6f6f529a380a1e770a3b27 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 15:49:46 -0700 Subject: [PATCH 11/15] Use fromJSON to evaluate string env variables as boolean --- .github/workflows/docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 161c84dc3..55830bf6d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,11 +49,11 @@ jobs: context: . file: ./docker/main.dockerfile build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }} - platforms: ${{ env.DOCKER_PUSH && 'linux/amd64,linux/arm64,linux/386' || 'linux/amd64' }} + platforms: ${{ fromJSON(env.DOCKER_PUSH) && 'linux/amd64,linux/arm64,linux/386' || 'linux/amd64' }} tags: ${{ steps.docker_meta_main.outputs.tags }} labels: ${{ steps.docker_meta_main.outputs.labels }} - push: ${{ env.DOCKER_PUSH }} - load: ${{ !env.DOCKER_PUSH }} + push: ${{ fromJSON(env.DOCKER_PUSH) }} + load: ${{ !fromJSON(env.DOCKER_PUSH) }} - name: Docker meta for cloud image id: docker_meta_cloud uses: crazy-max/ghaction-docker-meta@v1 @@ -65,7 +65,7 @@ jobs: context: . file: ./docker/cloud.dockerfile build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }} - platforms: ${{ env.DOCKER_PUSH && 'linux/amd64,linux/arm64,linux/386' || 'linux/amd64' }} + platforms: ${{ fromJSON(env.DOCKER_PUSH) && 'linux/amd64,linux/arm64,linux/386' || 'linux/amd64' }} tags: ${{ steps.docker_meta_cloud.outputs.tags }} labels: ${{ steps.docker_meta_cloud.outputs.labels }} - push: ${{ env.DOCKER_PUSH }} + push: ${{ fromJSON(env.DOCKER_PUSH) }} From ba36fafc1d6094c7116d54c4b48d0947c885e482 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 16:43:31 -0700 Subject: [PATCH 12/15] Inspect loaded image from main dockerfile --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 55830bf6d..45cfb9163 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,6 +54,9 @@ jobs: labels: ${{ steps.docker_meta_main.outputs.labels }} push: ${{ fromJSON(env.DOCKER_PUSH) }} load: ${{ !fromJSON(env.DOCKER_PUSH) }} + - name: Check images + run: | + docker images ls -a - name: Docker meta for cloud image id: docker_meta_cloud uses: crazy-max/ghaction-docker-meta@v1 From 6f4ea4ae21aa48daf0cce325047c13224ba3dc7f Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 17:01:52 -0700 Subject: [PATCH 13/15] Some more debug checks during inspect phase --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 45cfb9163..442793e83 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -56,7 +56,11 @@ jobs: load: ${{ !fromJSON(env.DOCKER_PUSH) }} - name: Check images run: | + df -h + docker buildx du --verbose docker images ls -a + docker image inspect ${{ steps.docker_meta_main.outputs.tags }} + - name: Docker meta for cloud image id: docker_meta_cloud uses: crazy-max/ghaction-docker-meta@v1 From 5969f9c6e4a6612d55dc0bb36d443e0977ad2bf3 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Wed, 22 Jun 2022 17:29:17 -0700 Subject: [PATCH 14/15] fix image list command --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 442793e83..afb3c16d5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,8 +57,7 @@ jobs: - name: Check images run: | df -h - docker buildx du --verbose - docker images ls -a + docker image ls docker image inspect ${{ steps.docker_meta_main.outputs.tags }} - name: Docker meta for cloud image From 3c6be773b64a1a562f2b581b90139811189974c2 Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 23 Jun 2022 10:53:53 -0700 Subject: [PATCH 15/15] Add debug prints after docker meta step --- .github/workflows/docker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index afb3c16d5..0ef3b6717 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,6 +65,11 @@ jobs: uses: crazy-max/ghaction-docker-meta@v1 with: images: nbraun/dask-sql-cloud + - name: Check images_2 + run: | + df -h + docker image ls + docker image inspect ${{ steps.docker_meta_main.outputs.tags }} - name: Build and push cloud image uses: docker/build-push-action@v2 with: