From d7fafea9befead769d4cf9131626ae6b1fd02efc Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 19:38:21 +0900 Subject: [PATCH 01/24] feat: use GHA to do build step-by-step and parallel --- .github/workflows/release.yml | 92 ++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3eb7ab1..dc24ca8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,19 +4,99 @@ on: tags: - '*' jobs: - release: - name: Publish a release + stage1: + name: Stage 1 runs-on: ubuntu-latest steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and publish images + - name: Set up version run: | - version="$(echo "${{ github.ref }}" | cut -d / -f 3)" - ./scripts/build -p . "$version" + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "tag=ghcr.io/toltec-dev/toolchain:$VERSION" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and puch toolchain + uses: docker/build-push-action@v5 + with: + context: toolchain/ + push: true + tags: ${{ env.tag }} + cache_from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILDKIT_INLINE_CACHE=1 + stage2: + name: Stage 2 + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "tag=ghcr.io/toltec-dev/base:$VERSION" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and puch toolchain + uses: docker/build-push-action@v5 + with: + context: toolchain/ + push: true + tags: ${{ env.tag }} + cache_from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILDKIT_INLINE_CACHE=1 + FROM=ghcr.io/toltec-dev/toolchain + stage3: + name: Stage 3 + runs-on: ubuntu-latest + needs: stage2 + strategy: + matrix: + from: [ghcr.io/toltec-dev/base] + target: [golang, python, qt, rust] + include: + - from: ghcr.io/toltec-dev/toolchain + - target: dotnet6 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "tag=ghcr.io/toltec-dev/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and puch toolchain + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.target }}/ + push: true + tags: ${{ env.tag }} + cache_from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILDKIT_INLINE_CACHE=1 + FROM=${{ matrix.from }} + From e14340e56b3a360c5e2a461e2a3cd0ce20d26891 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 19:38:50 +0900 Subject: [PATCH 02/24] temp: to do testing --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dc24ca8..4ddae04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/toltec-dev/toolchain:$VERSION" >> "$GITHUB_ENV" + echo "tag=ghcr.io/NoaHimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and puch toolchain @@ -48,7 +48,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/toltec-dev/base:$VERSION" >> "$GITHUB_ENV" + echo "tag=ghcr.io/NoaHimesaka1873/base:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and puch toolchain @@ -61,17 +61,17 @@ jobs: cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=ghcr.io/toltec-dev/toolchain + FROM=ghcr.io/NoaHimesaka1873/toolchain stage3: name: Stage 3 runs-on: ubuntu-latest needs: stage2 strategy: matrix: - from: [ghcr.io/toltec-dev/base] + from: [ghcr.io/NoaHimesaka1873/base] target: [golang, python, qt, rust] include: - - from: ghcr.io/toltec-dev/toolchain + - from: ghcr.io/NoaHimesaka1873/toolchain - target: dotnet6 steps: - name: Checkout the Git repository @@ -85,7 +85,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/toltec-dev/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" + echo "tag=ghcr.io/NoaHimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and puch toolchain From 3262b7b29f40cc0a5593f25c2706558c58c443b7 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 19:43:57 +0900 Subject: [PATCH 03/24] fix: typo --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ddae04..6673c9a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: context: toolchain/ push: true tags: ${{ env.tag }} - cache_from: type=gha + cache-from: type=gha cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 @@ -57,7 +57,7 @@ jobs: context: toolchain/ push: true tags: ${{ env.tag }} - cache_from: type=gha + cache-from: type=gha cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 @@ -94,7 +94,7 @@ jobs: context: ${{ matrix.target }}/ push: true tags: ${{ env.tag }} - cache_from: type=gha + cache-from: type=gha cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 From c9ee5c088c9dbb47c1f5411beaa0039c0aa2fc75 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 19:46:58 +0900 Subject: [PATCH 04/24] temp: fixing --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6673c9a..747efba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/NoaHimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" + echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and puch toolchain @@ -48,7 +48,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/NoaHimesaka1873/base:$VERSION" >> "$GITHUB_ENV" + echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and puch toolchain @@ -61,17 +61,17 @@ jobs: cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=ghcr.io/NoaHimesaka1873/toolchain + FROM=ghcr.io/noahimesaka1873/toolchain stage3: name: Stage 3 runs-on: ubuntu-latest needs: stage2 strategy: matrix: - from: [ghcr.io/NoaHimesaka1873/base] + from: [ghcr.io/noahimesaka1873/base] target: [golang, python, qt, rust] include: - - from: ghcr.io/NoaHimesaka1873/toolchain + - from: ghcr.io/noahimesaka1873/toolchain - target: dotnet6 steps: - name: Checkout the Git repository @@ -85,7 +85,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/NoaHimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" + echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and puch toolchain From 9ee872d94d939b7e0c16da7fee2df16eb083e090 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 21:41:08 +0900 Subject: [PATCH 05/24] fix: typo and missing tag --- .github/workflows/release.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 747efba..3af9da4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and puch toolchain + - name: Build and push toolchain uses: docker/build-push-action@v5 with: context: toolchain/ @@ -51,7 +51,7 @@ jobs: echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and puch toolchain + - name: Build and push toolchain uses: docker/build-push-action@v5 with: context: toolchain/ @@ -72,7 +72,7 @@ jobs: target: [golang, python, qt, rust] include: - from: ghcr.io/noahimesaka1873/toolchain - - target: dotnet6 + target: dotnet6 steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -88,7 +88,7 @@ jobs: echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and puch toolchain + - name: Build and push toolchain uses: docker/build-push-action@v5 with: context: ${{ matrix.target }}/ @@ -98,5 +98,4 @@ jobs: cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=${{ matrix.from }} - + FROM=${{ matrix.from }}:${{ env.tag }} From 36d11f6dd585543157b738d6c971bad242c16c41 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 22:43:33 +0900 Subject: [PATCH 06/24] feat: aarch64 build test 1 --- .github/workflows/release.yml | 143 +++++++++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3af9da4..b3702f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,12 @@ on: jobs: stage1: name: Stage 1 - runs-on: ubuntu-latest + runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + strategy: + matrix: + platform: + - linux/amd64 + - linux/arm64 steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -19,23 +24,84 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" + platform=${{ matrix.platform }} + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: toolchain/ push: true tags: ${{ env.tag }} cache-from: type=gha cache-to: type=gha,mode=max + outputs: type=image,"name=ghcr.io/noahimesaka1873/toolchain",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 + - name: Export digest + if: ${{ inputs.push_to_images != '' }} + run: | + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + - name: Upload digest + if: ${{ inputs.push_to_images != '' }} + uses: actions/upload-artifact@v4 + with: + name: digests-toolchain-${{ env.PLATFORM_PAIR }}-${{ env.VERSION }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + stage1-upload: + name: Stage 1 merge and push image + needs: stage1 + runs-on: ubuntu-24.04 + env: + PUSH_TO_IMAGES: ghcr.io/noahimesaka1873/toolchain + steps: + - uses: actions/checkout@v4 + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "VERSION=$VERSION" >> $GITHUB_ENV + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-toolchain-*-${{ env.VERSION }} + merge-multiple: true + - name: Log in to the GitHub Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.PUSH_TO_IMAGES }} + tags: ${{ env.VERSION }} + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + echo "$PUSH_TO_IMAGES" | xargs -I{} \ + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '{}@sha256:%s ' *) stage2: name: Stage 2 - runs-on: ubuntu-latest - needs: stage1 + needs: stage1-upload + runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + strategy: + matrix: + platform: + - linux/amd64 + - linux/arm64 steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -48,24 +114,80 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> "$GITHUB_ENV" + platform=${{ matrix.platform }} + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: toolchain/ push: true tags: ${{ env.tag }} cache-from: type=gha cache-to: type=gha,mode=max + outputs: type=image,"name=ghcr.io/noahimesaka1873/base",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 FROM=ghcr.io/noahimesaka1873/toolchain + - name: Export digest + if: ${{ inputs.push_to_images != '' }} + run: | + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + - name: Upload digest + if: ${{ inputs.push_to_images != '' }} + uses: actions/upload-artifact@v4 + with: + name: digests-base-${{ env.PLATFORM_PAIR }}-${{ env.VERSION }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + stage2-upload: + name: Stage 2 merge and push image + needs: stage2 + runs-on: ubuntu-24.04 + env: + PUSH_TO_IMAGES: ghcr.io/noahimesaka1873/base + steps: + - uses: actions/checkout@v4 + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "VERSION=$VERSION" >> $GITHUB_ENV + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-base-*-${{ env.VERSION }} + merge-multiple: true + - name: Log in to the GitHub Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.PUSH_TO_IMAGES }} + tags: ${{ env.VERSION }} + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + echo "$PUSH_TO_IMAGES" | xargs -I{} \ + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '{}@sha256:%s ' *) stage3: name: Stage 3 runs-on: ubuntu-latest - needs: stage2 + needs: stage2-upload strategy: matrix: from: [ghcr.io/noahimesaka1873/base] @@ -85,7 +207,8 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" + echo "version=$VERSION" >> $GITHUB_ENV + echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain @@ -98,4 +221,4 @@ jobs: cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=${{ matrix.from }}:${{ env.tag }} + FROM=${{ matrix.from }}:${{ env.version }} From 7b155e1a8b231051f8f8990297fbb311d2b57f7e Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 23:31:28 +0900 Subject: [PATCH 07/24] feat: aarch64 build test 2 --- .github/workflows/release.yml | 59 ++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b3702f6..5008926 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: with: context: toolchain/ push: true - tags: ${{ env.tag }} + platforms: ${{ matrix.platform }} cache-from: type=gha cache-to: type=gha,mode=max outputs: type=image,"name=ghcr.io/noahimesaka1873/toolchain",push-by-digest=true,name-canonical=true,push=true @@ -125,13 +125,13 @@ jobs: with: context: toolchain/ push: true - tags: ${{ env.tag }} + platforms: ${{ matrix.platform }} cache-from: type=gha cache-to: type=gha,mode=max outputs: type=image,"name=ghcr.io/noahimesaka1873/base",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=ghcr.io/noahimesaka1873/toolchain + FROM=ghcr.io/noahimesaka1873/toolchain:${{ env.VERSION }} - name: Export digest if: ${{ inputs.push_to_images != '' }} run: | @@ -186,15 +186,22 @@ jobs: $(printf '{}@sha256:%s ' *) stage3: name: Stage 3 - runs-on: ubuntu-latest + runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} needs: stage2-upload strategy: matrix: from: [ghcr.io/noahimesaka1873/base] target: [golang, python, qt, rust] + platform: + - linux/amd64 + - linux/arm64 include: - from: ghcr.io/noahimesaka1873/toolchain target: dotnet6 + platform: linux/amd64 + - from: ghcr.io/noahimesaka1873/toolchain + target: dotnet6 + platform: linux/arm64 steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -216,9 +223,51 @@ jobs: with: context: ${{ matrix.target }}/ push: true - tags: ${{ env.tag }} + platforms: ${{ matrix.platform }} cache-from: type=gha cache-to: type=gha,mode=max + outputs: type=image,"name=ghcr.io/noahimesaka1873/${{ matrix.target }}",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 FROM=${{ matrix.from }}:${{ env.version }} + stage3-upload: + name: Stage 3 merge and push image + needs: stage3 + runs-on: ubuntu-24.04 + strategy: + matrix: + target: [golang, python, qt, rust, dotnet6] + env: + PUSH_TO_IMAGES: ghcr.io/noahimesaka1873/${{ matrix.target }} + steps: + - uses: actions/checkout@v4 + - name: Set up version + run: | + VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" + echo "VERSION=$VERSION" >> $GITHUB_ENV + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-${{ matrix.target }}-*-${{ env.VERSION }} + merge-multiple: true + - name: Log in to the GitHub Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.PUSH_TO_IMAGES }} + tags: ${{ env.VERSION }} + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + echo "$PUSH_TO_IMAGES" | xargs -I{} \ + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '{}@sha256:%s ' *) From 21605d155815afe293dbfc285313a613872fe9bb Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Fri, 25 Apr 2025 23:37:06 +0900 Subject: [PATCH 08/24] feat: aarch64 build test 3 --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5008926..8bc9f02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,8 +36,8 @@ jobs: context: toolchain/ push: true platforms: ${{ matrix.platform }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=toolchain${{ env.PLATFORM_PAIR }} + cache-to: type=gha,mode=max,scope=toolchain outputs: type=image,"name=ghcr.io/noahimesaka1873/toolchain",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 @@ -126,8 +126,8 @@ jobs: context: toolchain/ push: true platforms: ${{ matrix.platform }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=base${{ env.PLATFORM_PAIR }} + cache-to: type=gha,mode=max,scope=base${{ env.PLATFORM_PAIR }} outputs: type=image,"name=ghcr.io/noahimesaka1873/base",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 @@ -224,8 +224,8 @@ jobs: context: ${{ matrix.target }}/ push: true platforms: ${{ matrix.platform }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.target }}${{ env.PLATFORM_PAIR }} + cache-to: type=gha,mode=max,scope=${{ matrix.target }}${{ env.PLATFORM_PAIR }} outputs: type=image,"name=ghcr.io/noahimesaka1873/${{ matrix.target }}",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 From 3f1dc6377bfbb08b98d6ff79a662b9a43957a60f Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 00:11:41 +0900 Subject: [PATCH 09/24] fix: remove ifs --- .github/workflows/release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bc9f02..dd6d894 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,13 +42,11 @@ jobs: build-args: | BUILDKIT_INLINE_CACHE=1 - name: Export digest - if: ${{ inputs.push_to_images != '' }} run: | mkdir -p "${{ runner.temp }}/digests" digest="${{ steps.build.outputs.digest }}" touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - if: ${{ inputs.push_to_images != '' }} uses: actions/upload-artifact@v4 with: name: digests-toolchain-${{ env.PLATFORM_PAIR }}-${{ env.VERSION }} @@ -133,13 +131,11 @@ jobs: BUILDKIT_INLINE_CACHE=1 FROM=ghcr.io/noahimesaka1873/toolchain:${{ env.VERSION }} - name: Export digest - if: ${{ inputs.push_to_images != '' }} run: | mkdir -p "${{ runner.temp }}/digests" digest="${{ steps.build.outputs.digest }}" touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - if: ${{ inputs.push_to_images != '' }} uses: actions/upload-artifact@v4 with: name: digests-base-${{ env.PLATFORM_PAIR }}-${{ env.VERSION }} From fdab65fc55e82f50fe0a158d527a6a2a03c788b0 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 00:13:51 +0900 Subject: [PATCH 10/24] fix: fix caching --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd6d894..78f12f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: push: true platforms: ${{ matrix.platform }} cache-from: type=gha,scope=toolchain${{ env.PLATFORM_PAIR }} - cache-to: type=gha,mode=max,scope=toolchain + cache-to: type=gha,mode=max,scope=toolchain${{ env.PLATFORM_PAIR }} outputs: type=image,"name=ghcr.io/noahimesaka1873/toolchain",push-by-digest=true,name-canonical=true,push=true build-args: | BUILDKIT_INLINE_CACHE=1 From b3194fc0333bb6ec47bff57ad46fdfc81bf81f71 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 00:59:28 +0900 Subject: [PATCH 11/24] Revert "feat: aarch64 build test 1" This reverts commit 36d11f6dd585543157b738d6c971bad242c16c41. --- .github/workflows/release.yml | 210 ++++------------------------------ 1 file changed, 21 insertions(+), 189 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78f12f2..3af9da4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,12 +6,7 @@ on: jobs: stage1: name: Stage 1 - runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} - strategy: - matrix: - platform: - - linux/amd64 - - linux/arm64 + runs-on: ubuntu-latest steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -24,82 +19,23 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - platform=${{ matrix.platform }} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> $GITHUB_ENV + echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v5 with: context: toolchain/ push: true - platforms: ${{ matrix.platform }} - cache-from: type=gha,scope=toolchain${{ env.PLATFORM_PAIR }} - cache-to: type=gha,mode=max,scope=toolchain${{ env.PLATFORM_PAIR }} - outputs: type=image,"name=ghcr.io/noahimesaka1873/toolchain",push-by-digest=true,name-canonical=true,push=true + tags: ${{ env.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - - name: Export digest - run: | - mkdir -p "${{ runner.temp }}/digests" - digest="${{ steps.build.outputs.digest }}" - touch "${{ runner.temp }}/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-toolchain-${{ env.PLATFORM_PAIR }}-${{ env.VERSION }} - path: ${{ runner.temp }}/digests/* - if-no-files-found: error - retention-days: 1 - stage1-upload: - name: Stage 1 merge and push image - needs: stage1 - runs-on: ubuntu-24.04 - env: - PUSH_TO_IMAGES: ghcr.io/noahimesaka1873/toolchain - steps: - - uses: actions/checkout@v4 - - name: Set up version - run: | - VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "VERSION=$VERSION" >> $GITHUB_ENV - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: ${{ runner.temp }}/digests - pattern: digests-toolchain-*-${{ env.VERSION }} - merge-multiple: true - - name: Log in to the GitHub Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.PUSH_TO_IMAGES }} - tags: ${{ env.VERSION }} - - name: Create manifest list and push - working-directory: ${{ runner.temp }}/digests - run: | - echo "$PUSH_TO_IMAGES" | xargs -I{} \ - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '{}@sha256:%s ' *) stage2: name: Stage 2 - needs: stage1-upload - runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} - strategy: - matrix: - platform: - - linux/amd64 - - linux/arm64 + runs-on: ubuntu-latest + needs: stage1 steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -112,92 +48,31 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - platform=${{ matrix.platform }} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> $GITHUB_ENV + echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v5 with: context: toolchain/ push: true - platforms: ${{ matrix.platform }} - cache-from: type=gha,scope=base${{ env.PLATFORM_PAIR }} - cache-to: type=gha,mode=max,scope=base${{ env.PLATFORM_PAIR }} - outputs: type=image,"name=ghcr.io/noahimesaka1873/base",push-by-digest=true,name-canonical=true,push=true + tags: ${{ env.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=ghcr.io/noahimesaka1873/toolchain:${{ env.VERSION }} - - name: Export digest - run: | - mkdir -p "${{ runner.temp }}/digests" - digest="${{ steps.build.outputs.digest }}" - touch "${{ runner.temp }}/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-base-${{ env.PLATFORM_PAIR }}-${{ env.VERSION }} - path: ${{ runner.temp }}/digests/* - if-no-files-found: error - retention-days: 1 - stage2-upload: - name: Stage 2 merge and push image - needs: stage2 - runs-on: ubuntu-24.04 - env: - PUSH_TO_IMAGES: ghcr.io/noahimesaka1873/base - steps: - - uses: actions/checkout@v4 - - name: Set up version - run: | - VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "VERSION=$VERSION" >> $GITHUB_ENV - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: ${{ runner.temp }}/digests - pattern: digests-base-*-${{ env.VERSION }} - merge-multiple: true - - name: Log in to the GitHub Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.PUSH_TO_IMAGES }} - tags: ${{ env.VERSION }} - - name: Create manifest list and push - working-directory: ${{ runner.temp }}/digests - run: | - echo "$PUSH_TO_IMAGES" | xargs -I{} \ - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '{}@sha256:%s ' *) + FROM=ghcr.io/noahimesaka1873/toolchain stage3: name: Stage 3 - runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} - needs: stage2-upload + runs-on: ubuntu-latest + needs: stage2 strategy: matrix: from: [ghcr.io/noahimesaka1873/base] target: [golang, python, qt, rust] - platform: - - linux/amd64 - - linux/arm64 include: - from: ghcr.io/noahimesaka1873/toolchain target: dotnet6 - platform: linux/amd64 - - from: ghcr.io/noahimesaka1873/toolchain - target: dotnet6 - platform: linux/arm64 steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -210,8 +85,7 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "version=$VERSION" >> $GITHUB_ENV - echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> $GITHUB_ENV + echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain @@ -219,51 +93,9 @@ jobs: with: context: ${{ matrix.target }}/ push: true - platforms: ${{ matrix.platform }} - cache-from: type=gha,scope=${{ matrix.target }}${{ env.PLATFORM_PAIR }} - cache-to: type=gha,mode=max,scope=${{ matrix.target }}${{ env.PLATFORM_PAIR }} - outputs: type=image,"name=ghcr.io/noahimesaka1873/${{ matrix.target }}",push-by-digest=true,name-canonical=true,push=true + tags: ${{ env.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=${{ matrix.from }}:${{ env.version }} - stage3-upload: - name: Stage 3 merge and push image - needs: stage3 - runs-on: ubuntu-24.04 - strategy: - matrix: - target: [golang, python, qt, rust, dotnet6] - env: - PUSH_TO_IMAGES: ghcr.io/noahimesaka1873/${{ matrix.target }} - steps: - - uses: actions/checkout@v4 - - name: Set up version - run: | - VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "VERSION=$VERSION" >> $GITHUB_ENV - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: ${{ runner.temp }}/digests - pattern: digests-${{ matrix.target }}-*-${{ env.VERSION }} - merge-multiple: true - - name: Log in to the GitHub Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.PUSH_TO_IMAGES }} - tags: ${{ env.VERSION }} - - name: Create manifest list and push - working-directory: ${{ runner.temp }}/digests - run: | - echo "$PUSH_TO_IMAGES" | xargs -I{} \ - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '{}@sha256:%s ' *) + FROM=${{ matrix.from }}:${{ env.tag }} From 4018965bd82bf3f2b460896964de4a14246f0635 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 01:03:00 +0900 Subject: [PATCH 12/24] fix: fix caching and version --- .github/workflows/release.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3af9da4..768e24d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,8 +28,8 @@ jobs: context: toolchain/ push: true tags: ${{ env.tag }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=toolchain + cache-to: type=gha,mode=max,scope=toolchain build-args: | BUILDKIT_INLINE_CACHE=1 stage2: @@ -49,6 +49,7 @@ jobs: run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> "$GITHUB_ENV" + echo "version=$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain @@ -57,11 +58,11 @@ jobs: context: toolchain/ push: true tags: ${{ env.tag }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=base + cache-to: type=gha,mode=max,scope=base build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=ghcr.io/noahimesaka1873/toolchain + FROM=ghcr.io/noahimesaka1873/toolchain:${{ env.version }} stage3: name: Stage 3 runs-on: ubuntu-latest @@ -86,6 +87,7 @@ jobs: run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" + echo "version=$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain @@ -94,8 +96,8 @@ jobs: context: ${{ matrix.target }}/ push: true tags: ${{ env.tag }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.target }} + cache-to: type=gha,mode=max,scope=${{ matrix.target }} build-args: | BUILDKIT_INLINE_CACHE=1 - FROM=${{ matrix.from }}:${{ env.tag }} + FROM=${{ matrix.from }}:${{ env.version }} From 4bd90b2fac812c25cd146a1fd4fd41155051a679 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 01:55:55 +0900 Subject: [PATCH 13/24] fix: typo --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 768e24d..289514e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,7 +55,7 @@ jobs: - name: Build and push toolchain uses: docker/build-push-action@v5 with: - context: toolchain/ + context: base/ push: true tags: ${{ env.tag }} cache-from: type=gha,scope=base From 3d4ac88ef3e4429b203be2e85f42f8bb2355ce76 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 14:46:20 +0900 Subject: [PATCH 14/24] feat: new script and GHA based on it --- .github/workflows/release.yml | 42 +------- scripts/build | 175 +++++++++++++++++----------------- scripts/lib | 30 ------ 3 files changed, 91 insertions(+), 156 deletions(-) delete mode 100644 scripts/lib diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 289514e..ff270a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,19 +19,11 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/noahimesaka1873/toolchain:$VERSION" >> "$GITHUB_ENV" + echo "version=$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v5 - with: - context: toolchain/ - push: true - tags: ${{ env.tag }} - cache-from: type=gha,scope=toolchain - cache-to: type=gha,mode=max,scope=toolchain - build-args: | - BUILDKIT_INLINE_CACHE=1 + run: ./scripts/build -p -g -s toolchain -v ${{ tag.version }} stage2: name: Stage 2 runs-on: ubuntu-latest @@ -48,32 +40,18 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/noahimesaka1873/base:$VERSION" >> "$GITHUB_ENV" echo "version=$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v5 - with: - context: base/ - push: true - tags: ${{ env.tag }} - cache-from: type=gha,scope=base - cache-to: type=gha,mode=max,scope=base - build-args: | - BUILDKIT_INLINE_CACHE=1 - FROM=ghcr.io/noahimesaka1873/toolchain:${{ env.version }} + run: ./scripts/build -p -g -s base -v ${{ tag.version }} stage3: name: Stage 3 runs-on: ubuntu-latest needs: stage2 strategy: matrix: - from: [ghcr.io/noahimesaka1873/base] - target: [golang, python, qt, rust] - include: - - from: ghcr.io/noahimesaka1873/toolchain - target: dotnet6 + target: [golang, python, qt, rust, dotnet6] steps: - name: Checkout the Git repository uses: actions/checkout@v3 @@ -86,18 +64,8 @@ jobs: - name: Set up version run: | VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)" - echo "tag=ghcr.io/noahimesaka1873/${{ matrix.target }}:$VERSION" >> "$GITHUB_ENV" echo "version=$VERSION" >> "$GITHUB_ENV" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.target }}/ - push: true - tags: ${{ env.tag }} - cache-from: type=gha,scope=${{ matrix.target }} - cache-to: type=gha,mode=max,scope=${{ matrix.target }} - build-args: | - BUILDKIT_INLINE_CACHE=1 - FROM=${{ matrix.from }}:${{ env.version }} + run: ./scripts/build -p -g -s ${{ matrix.target }} -v ${{ tag.version }} diff --git a/scripts/build b/scripts/build index c43fe74..65cd895 100755 --- a/scripts/build +++ b/scripts/build @@ -1,95 +1,92 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -set -e -source "${BASH_SOURCE%/*}"/lib +import argparse as ap +import subprocess as sp +import typing as tp +import os -usage="$0 [OPTION]... IMAGESDIR [VERSION] - -Build the Docker images under the IMAGESDIR directory. +parser = ap.ArgumentParser( + description=""" +Build the Docker images under the IMAGESDIR directory (default: .). Label each built image with the given VERSION number (default: latest). The build takes advantage of any reusable layer from previously built images. - -Options: - - -h Show this help message. - -p Publish each built image to the GitHub Container Registry - (you need to be logged in to ghcr.io with the appropriate - permissions for this to work)." - -helpflag= -publishflag= - -while getopts hp name; do - case $name in - h) helpflag=1 ;; - p) publishflag=1 ;; - *) error "Invalid option. Use the -h flag for more information." ;; - esac -done - -shift $((OPTIND - 1)) - -if [[ -n $helpflag ]]; then - echo "$usage" - exit -fi - -if [[ $# -eq 0 ]]; then - error "Missing IMAGESDIR arguments. Use the -h flag for more information." -fi - -if [[ $# -gt 2 ]]; then - error "Extraneous arguments. Use the -h flag for more information." -fi - -imagesdir="$1" -version="$2" - -# Enable BuildKit for better cache behavior -# See -export DOCKER_BUILDKIT=1 - -docker-build() { - from="$(image-name "$1" "$version")" - target="$(image-name "$2" "$version")" - status "Building image '$target'" - docker image build \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - --build-arg FROM="$from" \ - --cache-from "$(image-name "$2")" \ - --tag "$target" . - - if [[ -n $publishflag ]]; then - docker image push "$target" - fi +""" +) + +builds = ["toolchain", "base", "golang", "python", "qt", "rust", "dotnet6"] +requires = { + "toolchain": None, + "base": "toolchain", + "golang": "base", + "python": "base", + "qt": "base", + "rust": "base", + "dotnet6": "toolchain", } -pushd "$imagesdir"/toolchain -docker-build toolchain toolchain -popd - -pushd "$imagesdir"/base -docker-build toolchain base -popd - -pushd "$imagesdir"/golang -docker-build base golang -popd - -pushd "$imagesdir"/python -docker-build base python -popd - -pushd "$imagesdir"/qt -docker-build base qt -popd - -pushd "$imagesdir"/rust -docker-build base rust -popd - -pushd "$imagesdir"/dotnet6 -docker-build toolchain dotnet6 -popd - -status "Done" +parser.add_argument( + "-p", help="Publish built images to GitHub Container Registry", action="store_true" +) +parser.add_argument("-g", help="Use GitHub Actions for caching", action="store_true") +parser.add_argument("-d", help="Set image directory", default=".", metavar="IMAGESDIR") +parser.add_argument( + "-s", help="Just build single image given as argument", choices=builds +) +parser.add_argument( + "-v", help="Set version number", default="latest", metavar="VERSION" +) + + +def run_cmd(line: list[str]) -> None: + p = sp.Popen( + line, + stdout=sp.PIPE, + stderr=sp.STDOUT, + shell=True, + encoding="utf-8", + errors="replace", + ) + while True: + realtime_output = p.stdout.readline() + if realtime_output == "" and p.poll() is not None: + break + if realtime_output: + print(realtime_output.strip(), flush=True) + + +def build( + image: str, version: str, publish: bool, github: bool, _from: tp.Optional[str] = None +) -> None: + cmd: list[str] = [ + "docker", + "buildx", + "build", + "--build-arg", + "BUILDKIT_INLINE_CACHE=1", + ] + if _from: + cmd.extend(["--build-arg", f"FROM={_from}"]) + if github: + cmd.extend(["--cache-from", f"type=gha,scope={image}"]) + cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"]) + cmd.extend(["--tag", f"ghcr.io/noahimesaka1873/{image}:{version}", "."]) + run_cmd(cmd) + if publish: + cmd = ["docker", "push", f"ghcr.io/noahimesaka1873/{image}:{version}"] + run_cmd(cmd) + + +args = parser.parse_args() +curdir = os.getcwd() +os.chdir(args.d) +workdir = os.getcwd() +if args.s: + os.chdir(args.s) + build(args.s, args.v, args.p, args.g, requires[args.s]) + os.chdir(workdir) +else: + for i in builds: + os.chdir(i) + build(i, args.v, args.p, args.g, requires[i]) + os.chdir(workdir) +os.chdir(curdir) diff --git a/scripts/lib b/scripts/lib deleted file mode 100644 index b42ba69..0000000 --- a/scripts/lib +++ /dev/null @@ -1,30 +0,0 @@ -# vim: set ft=sh: - -# Print an error message to error output then exit -# -# Arguments: -# -# $1 - Error message to print -error() { - echo "Error: $1" >&2 - exit 1 -} - -# Print a status message -# -# Arguments: -# -# $1 - Current status message -status() { - echo "=> $1" >&2 -} - -# Get the full tag of a Docker image -# -# Arguments: -# -# $1 - Name of the image -# $2 - Version of the image -image-name() { - echo "ghcr.io/toltec-dev/$1:${2:-latest}" -} From e79b1ba8d1cd846da16ebdea0de9fb5fa36cba14 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 14:47:20 +0900 Subject: [PATCH 15/24] fix: typo --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff270a5..c893cac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - run: ./scripts/build -p -g -s toolchain -v ${{ tag.version }} + run: ./scripts/build -p -g -s toolchain -v ${{ env.version }} stage2: name: Stage 2 runs-on: ubuntu-latest @@ -44,7 +44,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - run: ./scripts/build -p -g -s base -v ${{ tag.version }} + run: ./scripts/build -p -g -s base -v ${{ env.version }} stage3: name: Stage 3 runs-on: ubuntu-latest @@ -68,4 +68,4 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push toolchain - run: ./scripts/build -p -g -s ${{ matrix.target }} -v ${{ tag.version }} + run: ./scripts/build -p -g -s ${{ matrix.target }} -v ${{ env.version }} From 1166a668a8a2eca38c8eb69105653bc48699aabf Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 14:57:56 +0900 Subject: [PATCH 16/24] fix: script fixes --- scripts/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build b/scripts/build index 65cd895..9e64a84 100755 --- a/scripts/build +++ b/scripts/build @@ -42,7 +42,6 @@ def run_cmd(line: list[str]) -> None: line, stdout=sp.PIPE, stderr=sp.STDOUT, - shell=True, encoding="utf-8", errors="replace", ) @@ -86,6 +85,7 @@ if args.s: os.chdir(workdir) else: for i in builds: + print(f"Building {i}...") os.chdir(i) build(i, args.v, args.p, args.g, requires[i]) os.chdir(workdir) From a8b37d0aad19a71a872f84018be54f95111e0b69 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 15:54:23 +0900 Subject: [PATCH 17/24] fix: script fixes --- scripts/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build b/scripts/build index 9e64a84..eb37b29 100755 --- a/scripts/build +++ b/scripts/build @@ -64,7 +64,7 @@ def build( "BUILDKIT_INLINE_CACHE=1", ] if _from: - cmd.extend(["--build-arg", f"FROM={_from}"]) + cmd.extend(["--build-arg", f"FROM=ghcr.io/noahimesaka1873/{_from}:{version}"]) if github: cmd.extend(["--cache-from", f"type=gha,scope={image}"]) cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"]) From b9ecd72f9d1379feb9b9e02f8e18e865f794cb9e Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 16:47:41 +0900 Subject: [PATCH 18/24] fix: script fixes --- scripts/build | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build b/scripts/build index eb37b29..e1000c8 100755 --- a/scripts/build +++ b/scripts/build @@ -68,11 +68,10 @@ def build( if github: cmd.extend(["--cache-from", f"type=gha,scope={image}"]) cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"]) + if publish: + cmd.extend(["--push"]) cmd.extend(["--tag", f"ghcr.io/noahimesaka1873/{image}:{version}", "."]) run_cmd(cmd) - if publish: - cmd = ["docker", "push", f"ghcr.io/noahimesaka1873/{image}:{version}"] - run_cmd(cmd) args = parser.parse_args() From 916f801c622e23759ffad01b715469921f141ea5 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 17:42:11 +0900 Subject: [PATCH 19/24] fix: exit w/ error if build errored out --- scripts/build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build b/scripts/build index e1000c8..37c949c 100755 --- a/scripts/build +++ b/scripts/build @@ -51,6 +51,8 @@ def run_cmd(line: list[str]) -> None: break if realtime_output: print(realtime_output.strip(), flush=True) + if p.returncode != 0: + exit(p.returncode) def build( From 54b4079458b88e3755921a9a7db82b8786f2c3b3 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 17:42:37 +0900 Subject: [PATCH 20/24] revert test and make it ready for PR --- scripts/build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build b/scripts/build index 37c949c..5c01751 100755 --- a/scripts/build +++ b/scripts/build @@ -66,13 +66,13 @@ def build( "BUILDKIT_INLINE_CACHE=1", ] if _from: - cmd.extend(["--build-arg", f"FROM=ghcr.io/noahimesaka1873/{_from}:{version}"]) + cmd.extend(["--build-arg", f"FROM=ghcr.io/toltec-dev/{_from}:{version}"]) if github: cmd.extend(["--cache-from", f"type=gha,scope={image}"]) cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"]) if publish: cmd.extend(["--push"]) - cmd.extend(["--tag", f"ghcr.io/noahimesaka1873/{image}:{version}", "."]) + cmd.extend(["--tag", f"ghcr.io/toltec-dev/{image}:{version}", "."]) run_cmd(cmd) From 8e4fe262a480c1df6b8d2711464ab6caf70b5d76 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sat, 26 Apr 2025 17:46:39 +0900 Subject: [PATCH 21/24] use new build process for build and pr too --- .github/workflows/build.yml | 50 +++++++++++++++++++++++++++++---- .github/workflows/pr.yml | 56 ++++++++++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 21f4370..de93d0d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,18 +6,56 @@ on: - v2.x - v3.x jobs: - build: - name: Publish latest + stage1: + name: Stage 1 runs-on: ubuntu-latest steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and publish images - run: | - ./scripts/build -p . + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s toolchain + stage2: + name: Stage 2 + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s base + stage3: + name: Stage 3 + runs-on: ubuntu-latest + needs: stage2 + strategy: + matrix: + target: [golang, python, qt, rust, dotnet6] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -p -g -s ${{ matrix.target }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1c61eee..4a5ca5b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -6,14 +6,56 @@ on: - v2.x - v3.x jobs: - build: - name: Publish latest + stage1: + name: Stage 1 runs-on: ubuntu-latest - permissions: - contents: read steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Build images - run: | - ./scripts/build . + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -g -s toolchain + stage2: + name: Stage 2 + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -g -s base + stage3: + name: Stage 3 + runs-on: ubuntu-latest + needs: stage2 + strategy: + matrix: + target: [golang, python, qt, rust, dotnet6] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push toolchain + run: ./scripts/build -g -s ${{ matrix.target }} From 1ffd465b28bea0c921eeff2aabead020228a20d2 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Sat, 26 Apr 2025 12:45:32 -0600 Subject: [PATCH 22/24] Update pr.yml --- .github/workflows/pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4a5ca5b..d62326c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -5,6 +5,7 @@ on: - v1.x - v2.x - v3.x + - v4.x jobs: stage1: name: Stage 1 From 8f57a468b5ec125a95c06f0cc23da16daf95b992 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Sat, 26 Apr 2025 12:46:00 -0600 Subject: [PATCH 23/24] Update build.yml --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de93d0d..5d1b194 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,7 @@ on: - v1.x - v2.x - v3.x + - v4.x jobs: stage1: name: Stage 1 From 66b193cbd88a7d388567cb7b44606dbd450ad8e4 Mon Sep 17 00:00:00 2001 From: Noa Himesaka Date: Sun, 27 Apr 2025 12:06:20 +0900 Subject: [PATCH 24/24] fix: use curl -L to follow redirects --- base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/Dockerfile b/base/Dockerfile index 48fcd71..7f5ebf0 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -174,7 +174,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && cd /root \ && mkdir openssl \ && cd openssl \ - && curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -o openssl.tar.gz \ + && curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -Lo openssl.tar.gz \ && echo "ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46 openssl.tar.gz" > sha256sums \ && sha256sum -c sha256sums \ && tar --strip-components=1 -xf openssl.tar.gz \