Multiarch native build (#270)

febus982 · web-flow · commit f8668b13a232 · 2025-03-06T18:05:42.000Z
* Build multiarch images with native workers (removes QEMU)

* Switch to registry based cache for better docker buildx support
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
@@ -24,10 +24,30 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
+      # GitHub gives only repository complete in <owner>/<repo> format.
+      # Need some manual shenanigans
+      # Set IMAGE_NAME so we can push to <owner>/<repo>/<image>
+      # Transform os/arch to os-arch for suffix target
+      - name: Set ENV variables
+        run: |
+          echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV
+
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      # Login against a Docker registry
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3.3.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       # This might be unnecessary as tests are not
       # multiplatform
       - name: Setup Docker buildx
@@ -43,17 +63,18 @@ jobs:
           load: true
           target: dev
           tags: ${{ env.TEST_TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:tests
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:tests,mode=max
 
       # This is a barrier check to make sure we push a functional
       # docker image, we can avoid linting
       - name: Run tests in the test image
         run: |
           docker run --rm ${{ env.TEST_TAG }} make ci-test
 
-  build:
-    runs-on: ubuntu-latest
+  # Inspired to https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
+  build-arch:
+    runs-on: ${{ matrix.arch.runner }}
     needs: test
     permissions:
       contents: read
@@ -63,24 +84,120 @@ jobs:
       id-token: write
     strategy:
       matrix:
+        arch:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            # There is no latest for ARM yet
+            # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+            runner: ubuntu-24.04-arm
         docker_target:
           - migrations
           - http
           - socketio
           - dramatiq
     steps:
       # GitHub gives only repository complete in <owner>/<repo> format.
-      # Need some manual sheanigans
+      # Need some manual shenanigans
       # Set IMAGE_NAME so we can push to <owner>/<repo>/<image>
+      # Transform os/arch to os-arch for suffix target
       - name: Set ENV variables
         run: |
           echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV
+          platform=${{ matrix.arch.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      # Install the cosign tool
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3.8.1
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v3.10.0
+
+      # Login against a Docker registry
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3.3.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # We extract metadata without tags for single image
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5.7.0
+        with:
+          # list of Docker images to use as base name for tags
+          # <registry/<owner>/<repo_name>/<repo_name>-<target>
+          images: |
+            ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }}
+
+
+      # This build an image WITHOUT tags and outputs the digests, so that we can aggragate them later
+      - name: Build and push production image
+        id: build-and-push
+        uses: docker/build-push-action@v6.15.0
+        with:
+          context: .
+          target: ${{ matrix.docker_target }}
+          platforms: ${{ matrix.arch.platform }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
+          tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:buildcache-${{ matrix.docker_target }}-${{ env.PLATFORM_PAIR }}
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:buildcache-${{ matrix.docker_target }}-${{ env.PLATFORM_PAIR }},mode=max
+
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests/${{ matrix.docker_target }}
+          digest="${{ steps.build-and-push.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${{ matrix.docker_target }}/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/${{ matrix.docker_target }}/*
+          if-no-files-found: error
+          retention-days: 1
+
+
+  aggregate-manifests:
+    runs-on: ubuntu-latest
+    needs: build-arch
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+    strategy:
+      matrix:
+        docker_target:
+          - migrations
+          - http
+          - socketio
+          - dramatiq
+
+    steps:
+      # GitHub gives only repository complete in <owner>/<repo> format.
+      # Need some manual sheanigans
+      # Set IMAGE_NAME so we can push to <owner>/<repo>/<image>
+      - name: Set ENV variables
+        run: |
+          echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests/${{ matrix.docker_target }}
+          pattern: digests-*
+          merge-multiple: true
 
       # Install the cosign tool
       # https://github.com/sigstore/cosign-installer
@@ -115,20 +232,15 @@ jobs:
             type=raw,value={{branch}}-latest
             type=raw,value={{branch}}-{{date 'YYYYMMDDHHmmss'}}
 
-      # Build and push Docker image with Buildx
-      # https://github.com/docker/build-push-action
-      - name: Build and push production image
-        id: build-and-push
-        uses: docker/build-push-action@v6.15.0
-        with:
-          context: .
-          target: ${{ matrix.docker_target }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests/${{ matrix.docker_target }}
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }}:${{ steps.meta.outputs.version }}
 
       #TODO: Implement signature using generated key: https://docs.sigstore.dev/signing/quickstart/#signing-with-a-generated-key
 
@@ -141,12 +253,11 @@ jobs:
         env:
           # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
           TAGS: ${{ steps.meta.outputs.tags }}
-          DIGEST: ${{ steps.build-and-push.outputs.digest }}
         # This step uses the identity token to provision an ephemeral certificate
         # against the sigstore community Fulcio instance.
         run: |
           images=""
           for tag in ${TAGS}; do
-            images+="${tag}@${DIGEST} "
+            images+="${tag}@$(docker buildx imagetools inspect --format '{{json .Manifest.Digest}}' ${tag} | xargs) "
           done
           cosign sign --yes ${images}
diff --git a/Dockerfile b/Dockerfile
@@ -14,22 +14,25 @@ RUN mkdir /venv && chown nonroot:nonroot /venv
 ENV PATH="/venv/bin:$PATH"
 
 # Install necessary runtime libraries (e.g. libmysql)
-RUN apt-get update \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked  \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update \
     && apt-get install -y --no-install-recommends \
-    make \
-    && rm -rf /var/lib/apt/lists/*
+    make
 
 FROM base AS base_builder
 ENV UV_PROJECT_ENVIRONMENT=/venv
 # Enable bytecode compilation
 ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
 
 # Install build system requirements (gcc, library headers, etc.)
 # for compiled Python requirements like psycopg2
-RUN apt-get update \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked  \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update \
     && apt-get install -y --no-install-recommends \
-    build-essential gcc git \
-    && rm -rf /var/lib/apt/lists/*
+    build-essential gcc git
 
 COPY --from=ghcr.io/astral-sh/uv:0.6.3 /uv /uvx /bin/
 
@@ -44,7 +47,7 @@ COPY --chown=nonroot:nonroot Makefile .
 # Dev image, contains all files and dependencies
 FROM base_builder AS dev
 COPY --chown=nonroot:nonroot . .
-RUN --mount=type=cache,target=~/.cache/uv \
+RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \
     make dev-dependencies
 
 # Note that opentelemetry doesn't play well together with uvicorn reloader
@@ -53,22 +56,22 @@ CMD ["uvicorn", "http_app:create_app", "--host", "0.0.0.0", "--port", "8000", "-
 
 # Installs requirements to run production dramatiq application
 FROM base_builder AS dramatiq_builder
-RUN --mount=type=cache,target=~/.cache/uv \
+RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \
     uv sync --no-dev --no-install-project --frozen --no-editable
 
 # Installs requirements to run production http application
 FROM base_builder AS http_builder
-RUN --mount=type=cache,target=~/.cache/uv \
+RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \
     uv sync --no-dev --group http --no-install-project --frozen --no-editable
 
 # Installs requirements to run production socketio application
 FROM base_builder AS socketio_builder
-RUN --mount=type=cache,target=~/.cache/uv \
+RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \
     uv sync --no-dev --group socketio --no-install-project --frozen --no-editable
 
 # Installs requirements to run production migrations application
 FROM base_builder AS migrations_builder
-RUN --mount=type=cache,target=~/.cache/uv \
+RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \
     uv sync --no-dev --group migrations --no-install-project --frozen --no-editable
 
 # Create the base app with the common python packages
