Skip to content

Commit 4b5a6c6

Browse files
Staged builds (#40)
* feat: Update to use staged builds * fix: use curl -L to follow redirects --------- Co-authored-by: Nathaniel van Diepen <Eeems@users.noreply.github.com>
1 parent b4259ac commit 4b5a6c6

File tree

6 files changed

+238
-139
lines changed

6 files changed

+238
-139
lines changed

.github/workflows/build.yml

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,58 @@ on:
55
- v1.x
66
- v2.x
77
- v3.x
8+
- v4.x
89
jobs:
9-
build:
10-
name: Publish latest
10+
stage1:
11+
name: Stage 1
1112
runs-on: ubuntu-latest
1213
steps:
1314
- name: Checkout the Git repository
1415
uses: actions/checkout@v3
15-
- name: Login to Docker Hub
16+
- name: Login to GitHub Container Registry
1617
uses: docker/login-action@v2
1718
with:
1819
registry: ghcr.io
1920
username: ${{ github.actor }}
2021
password: ${{ secrets.GITHUB_TOKEN }}
21-
- name: Build and publish images
22-
run: |
23-
./scripts/build -p .
22+
- name: Set up Docker Buildx
23+
uses: docker/setup-buildx-action@v3
24+
- name: Build and push toolchain
25+
run: ./scripts/build -p -g -s toolchain
26+
stage2:
27+
name: Stage 2
28+
runs-on: ubuntu-latest
29+
needs: stage1
30+
steps:
31+
- name: Checkout the Git repository
32+
uses: actions/checkout@v3
33+
- name: Login to GitHub Container Registry
34+
uses: docker/login-action@v2
35+
with:
36+
registry: ghcr.io
37+
username: ${{ github.actor }}
38+
password: ${{ secrets.GITHUB_TOKEN }}
39+
- name: Set up Docker Buildx
40+
uses: docker/setup-buildx-action@v3
41+
- name: Build and push toolchain
42+
run: ./scripts/build -p -g -s base
43+
stage3:
44+
name: Stage 3
45+
runs-on: ubuntu-latest
46+
needs: stage2
47+
strategy:
48+
matrix:
49+
target: [golang, python, qt, rust, dotnet6]
50+
steps:
51+
- name: Checkout the Git repository
52+
uses: actions/checkout@v3
53+
- name: Login to GitHub Container Registry
54+
uses: docker/login-action@v2
55+
with:
56+
registry: ghcr.io
57+
username: ${{ github.actor }}
58+
password: ${{ secrets.GITHUB_TOKEN }}
59+
- name: Set up Docker Buildx
60+
uses: docker/setup-buildx-action@v3
61+
- name: Build and push toolchain
62+
run: ./scripts/build -p -g -s ${{ matrix.target }}

.github/workflows/pr.yml

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,58 @@ on:
55
- v1.x
66
- v2.x
77
- v3.x
8+
- v4.x
89
jobs:
9-
build:
10-
name: Publish latest
10+
stage1:
11+
name: Stage 1
1112
runs-on: ubuntu-latest
12-
permissions:
13-
contents: read
1413
steps:
1514
- name: Checkout the Git repository
1615
uses: actions/checkout@v3
17-
- name: Build images
18-
run: |
19-
./scripts/build .
16+
- name: Login to GitHub Container Registry
17+
uses: docker/login-action@v2
18+
with:
19+
registry: ghcr.io
20+
username: ${{ github.actor }}
21+
password: ${{ secrets.GITHUB_TOKEN }}
22+
- name: Set up Docker Buildx
23+
uses: docker/setup-buildx-action@v3
24+
- name: Build and push toolchain
25+
run: ./scripts/build -g -s toolchain
26+
stage2:
27+
name: Stage 2
28+
runs-on: ubuntu-latest
29+
needs: stage1
30+
steps:
31+
- name: Checkout the Git repository
32+
uses: actions/checkout@v3
33+
- name: Login to GitHub Container Registry
34+
uses: docker/login-action@v2
35+
with:
36+
registry: ghcr.io
37+
username: ${{ github.actor }}
38+
password: ${{ secrets.GITHUB_TOKEN }}
39+
- name: Set up Docker Buildx
40+
uses: docker/setup-buildx-action@v3
41+
- name: Build and push toolchain
42+
run: ./scripts/build -g -s base
43+
stage3:
44+
name: Stage 3
45+
runs-on: ubuntu-latest
46+
needs: stage2
47+
strategy:
48+
matrix:
49+
target: [golang, python, qt, rust, dotnet6]
50+
steps:
51+
- name: Checkout the Git repository
52+
uses: actions/checkout@v3
53+
- name: Login to GitHub Container Registry
54+
uses: docker/login-action@v2
55+
with:
56+
registry: ghcr.io
57+
username: ${{ github.actor }}
58+
password: ${{ secrets.GITHUB_TOKEN }}
59+
- name: Set up Docker Buildx
60+
uses: docker/setup-buildx-action@v3
61+
- name: Build and push toolchain
62+
run: ./scripts/build -g -s ${{ matrix.target }}

.github/workflows/release.yml

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,68 @@ on:
44
tags:
55
- '*'
66
jobs:
7-
release:
8-
name: Publish a release
7+
stage1:
8+
name: Stage 1
99
runs-on: ubuntu-latest
1010
steps:
1111
- name: Checkout the Git repository
1212
uses: actions/checkout@v3
13-
- name: Login to Docker Hub
13+
- name: Login to GitHub Container Registry
1414
uses: docker/login-action@v2
1515
with:
1616
registry: ghcr.io
1717
username: ${{ github.actor }}
1818
password: ${{ secrets.GITHUB_TOKEN }}
19-
- name: Build and publish images
19+
- name: Set up version
2020
run: |
21-
version="$(echo "${{ github.ref }}" | cut -d / -f 3)"
22-
./scripts/build -p . "$version"
21+
VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)"
22+
echo "version=$VERSION" >> "$GITHUB_ENV"
23+
- name: Set up Docker Buildx
24+
uses: docker/setup-buildx-action@v3
25+
- name: Build and push toolchain
26+
run: ./scripts/build -p -g -s toolchain -v ${{ env.version }}
27+
stage2:
28+
name: Stage 2
29+
runs-on: ubuntu-latest
30+
needs: stage1
31+
steps:
32+
- name: Checkout the Git repository
33+
uses: actions/checkout@v3
34+
- name: Login to GitHub Container Registry
35+
uses: docker/login-action@v2
36+
with:
37+
registry: ghcr.io
38+
username: ${{ github.actor }}
39+
password: ${{ secrets.GITHUB_TOKEN }}
40+
- name: Set up version
41+
run: |
42+
VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)"
43+
echo "version=$VERSION" >> "$GITHUB_ENV"
44+
- name: Set up Docker Buildx
45+
uses: docker/setup-buildx-action@v3
46+
- name: Build and push toolchain
47+
run: ./scripts/build -p -g -s base -v ${{ env.version }}
48+
stage3:
49+
name: Stage 3
50+
runs-on: ubuntu-latest
51+
needs: stage2
52+
strategy:
53+
matrix:
54+
target: [golang, python, qt, rust, dotnet6]
55+
steps:
56+
- name: Checkout the Git repository
57+
uses: actions/checkout@v3
58+
- name: Login to GitHub Container Registry
59+
uses: docker/login-action@v2
60+
with:
61+
registry: ghcr.io
62+
username: ${{ github.actor }}
63+
password: ${{ secrets.GITHUB_TOKEN }}
64+
- name: Set up version
65+
run: |
66+
VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)"
67+
echo "version=$VERSION" >> "$GITHUB_ENV"
68+
- name: Set up Docker Buildx
69+
uses: docker/setup-buildx-action@v3
70+
- name: Build and push toolchain
71+
run: ./scripts/build -p -g -s ${{ matrix.target }} -v ${{ env.version }}

base/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
174174
&& cd /root \
175175
&& mkdir openssl \
176176
&& cd openssl \
177-
&& curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -o openssl.tar.gz \
177+
&& curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -Lo openssl.tar.gz \
178178
&& echo "ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46 openssl.tar.gz" > sha256sums \
179179
&& sha256sum -c sha256sums \
180180
&& tar --strip-components=1 -xf openssl.tar.gz \

scripts/build

Lines changed: 87 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,93 @@
1-
#!/usr/bin/env bash
1+
#!/usr/bin/env python3
22

3-
set -e
4-
source "${BASH_SOURCE%/*}"/lib
3+
import argparse as ap
4+
import subprocess as sp
5+
import typing as tp
6+
import os
57

6-
usage="$0 [OPTION]... IMAGESDIR [VERSION]
7-
8-
Build the Docker images under the IMAGESDIR directory.
8+
parser = ap.ArgumentParser(
9+
description="""
10+
Build the Docker images under the IMAGESDIR directory (default: .).
911
Label each built image with the given VERSION number (default: latest).
1012
The build takes advantage of any reusable layer from previously built images.
11-
12-
Options:
13-
14-
-h Show this help message.
15-
-p Publish each built image to the GitHub Container Registry
16-
(you need to be logged in to ghcr.io with the appropriate
17-
permissions for this to work)."
18-
19-
helpflag=
20-
publishflag=
21-
22-
while getopts hp name; do
23-
case $name in
24-
h) helpflag=1 ;;
25-
p) publishflag=1 ;;
26-
*) error "Invalid option. Use the -h flag for more information." ;;
27-
esac
28-
done
29-
30-
shift $((OPTIND - 1))
31-
32-
if [[ -n $helpflag ]]; then
33-
echo "$usage"
34-
exit
35-
fi
36-
37-
if [[ $# -eq 0 ]]; then
38-
error "Missing IMAGESDIR arguments. Use the -h flag for more information."
39-
fi
40-
41-
if [[ $# -gt 2 ]]; then
42-
error "Extraneous arguments. Use the -h flag for more information."
43-
fi
44-
45-
imagesdir="$1"
46-
version="$2"
47-
48-
# Enable BuildKit for better cache behavior
49-
# See <https://docs.docker.com/engine/reference/builder/#buildkit>
50-
export DOCKER_BUILDKIT=1
51-
52-
docker-build() {
53-
from="$(image-name "$1" "$version")"
54-
target="$(image-name "$2" "$version")"
55-
status "Building image '$target'"
56-
docker image build \
57-
--build-arg BUILDKIT_INLINE_CACHE=1 \
58-
--build-arg FROM="$from" \
59-
--cache-from "$(image-name "$2")" \
60-
--tag "$target" .
61-
62-
if [[ -n $publishflag ]]; then
63-
docker image push "$target"
64-
fi
13+
"""
14+
)
15+
16+
builds = ["toolchain", "base", "golang", "python", "qt", "rust", "dotnet6"]
17+
requires = {
18+
"toolchain": None,
19+
"base": "toolchain",
20+
"golang": "base",
21+
"python": "base",
22+
"qt": "base",
23+
"rust": "base",
24+
"dotnet6": "toolchain",
6525
}
6626

67-
pushd "$imagesdir"/toolchain
68-
docker-build toolchain toolchain
69-
popd
70-
71-
pushd "$imagesdir"/base
72-
docker-build toolchain base
73-
popd
74-
75-
pushd "$imagesdir"/golang
76-
docker-build base golang
77-
popd
78-
79-
pushd "$imagesdir"/python
80-
docker-build base python
81-
popd
82-
83-
pushd "$imagesdir"/qt
84-
docker-build base qt
85-
popd
86-
87-
pushd "$imagesdir"/rust
88-
docker-build base rust
89-
popd
90-
91-
pushd "$imagesdir"/dotnet6
92-
docker-build toolchain dotnet6
93-
popd
94-
95-
status "Done"
27+
parser.add_argument(
28+
"-p", help="Publish built images to GitHub Container Registry", action="store_true"
29+
)
30+
parser.add_argument("-g", help="Use GitHub Actions for caching", action="store_true")
31+
parser.add_argument("-d", help="Set image directory", default=".", metavar="IMAGESDIR")
32+
parser.add_argument(
33+
"-s", help="Just build single image given as argument", choices=builds
34+
)
35+
parser.add_argument(
36+
"-v", help="Set version number", default="latest", metavar="VERSION"
37+
)
38+
39+
40+
def run_cmd(line: list[str]) -> None:
41+
p = sp.Popen(
42+
line,
43+
stdout=sp.PIPE,
44+
stderr=sp.STDOUT,
45+
encoding="utf-8",
46+
errors="replace",
47+
)
48+
while True:
49+
realtime_output = p.stdout.readline()
50+
if realtime_output == "" and p.poll() is not None:
51+
break
52+
if realtime_output:
53+
print(realtime_output.strip(), flush=True)
54+
if p.returncode != 0:
55+
exit(p.returncode)
56+
57+
58+
def build(
59+
image: str, version: str, publish: bool, github: bool, _from: tp.Optional[str] = None
60+
) -> None:
61+
cmd: list[str] = [
62+
"docker",
63+
"buildx",
64+
"build",
65+
"--build-arg",
66+
"BUILDKIT_INLINE_CACHE=1",
67+
]
68+
if _from:
69+
cmd.extend(["--build-arg", f"FROM=ghcr.io/toltec-dev/{_from}:{version}"])
70+
if github:
71+
cmd.extend(["--cache-from", f"type=gha,scope={image}"])
72+
cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"])
73+
if publish:
74+
cmd.extend(["--push"])
75+
cmd.extend(["--tag", f"ghcr.io/toltec-dev/{image}:{version}", "."])
76+
run_cmd(cmd)
77+
78+
79+
args = parser.parse_args()
80+
curdir = os.getcwd()
81+
os.chdir(args.d)
82+
workdir = os.getcwd()
83+
if args.s:
84+
os.chdir(args.s)
85+
build(args.s, args.v, args.p, args.g, requires[args.s])
86+
os.chdir(workdir)
87+
else:
88+
for i in builds:
89+
print(f"Building {i}...")
90+
os.chdir(i)
91+
build(i, args.v, args.p, args.g, requires[i])
92+
os.chdir(workdir)
93+
os.chdir(curdir)

0 commit comments

Comments
 (0)