Skip to content

Commit 1af7be7

Browse files
eranrunderanrund
committed
Switch the nightly build to producing a multiarch docker image that supports both x86_64 and arm64
1 parent e13ed1f commit 1af7be7

File tree

5 files changed

+305
-24
lines changed

5 files changed

+305
-24
lines changed

‎.github/workflows/nightly.yml‎

Lines changed: 125 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,50 +9,156 @@ on:
99
branches:
1010
- 'main'
1111

12+
env:
13+
REGISTRY_IMAGE: clux/muslrust
14+
1215
jobs:
13-
docker:
16+
build:
1417
name: 'Nightly Build'
1518
runs-on: 'ubuntu-latest'
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
platform: [linux/amd64, linux/arm64]
23+
include:
24+
- platform: linux/amd64
25+
dockerfile: Dockerfile.x86_64
26+
arch: amd64
27+
target_dir: x86_64-unknown-linux-musl
28+
- platform: linux/arm64
29+
dockerfile: Dockerfile.arm64
30+
arch: arm64
31+
target_dir: aarch64-unknown-linux-musl
1632
steps:
1733
- uses: 'actions/checkout@v2'
1834
- uses: extractions/setup-just@v1
1935

2036
- name: Login to DockerHub
21-
if: github.event_name != 'pull_request'
2237
uses: docker/login-action@v1
2338
with:
2439
username: clux
2540
password: ${{ secrets.DOCKERHUB_TOKEN }}
2641

42+
- name: Prepare
43+
run: |
44+
platform=${{ matrix.platform }}
45+
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
46+
47+
- name: Docker meta
48+
id: meta
49+
uses: docker/metadata-action@v5
50+
with:
51+
images: ${{ env.REGISTRY_IMAGE }}
52+
53+
- name: Set up QEMU
54+
uses: docker/setup-qemu-action@v3
55+
56+
- name: Set up Docker Buildx
57+
uses: docker/setup-buildx-action@v3
58+
2759
- name: Build nightly image
28-
uses: docker/build-push-action@v2
60+
id: build
61+
uses: docker/build-push-action@v3
2962
with:
3063
context: .
64+
platforms: ${{ matrix.platform }}
65+
labels: ${{ steps.meta.outputs.labels }}
66+
file: ${{ matrix.dockerfile }}
3167
push: false
68+
load: true
69+
tags: rustmusl-temp
3270
build-args: |
3371
CHANNEL=nightly
34-
tags: clux/muslrust:temp
3572
36-
- name: Compute tags
73+
- name: Run tests
74+
shell: bash
75+
run: |
76+
docker image ls
77+
docker buildx build --platform ${{ matrix.platform }} --output type=docker -t test-runner - < Dockerfile.test-runner
78+
docker image ls
79+
TARGET_DIR=${{ matrix.target_dir }} PLATFORM=${{ matrix.platform }} just test
80+
81+
# The date/channel/version are expected to be the same on both architectures and are needed for the merge step.
82+
# We store them here since it makes the merge step a bit easier - it doesn't need to figure out which of the
83+
# architectures it can run (to extract the rust version). The problem is that it appears we can't run images
84+
# that were built by docker buildx (the build-push-action step) locally. They get pushed to dockerhub but are
85+
# only identifiable by their digest and it appears docker does not let us select an image that way.
86+
# Not the most elegant, but it works.
87+
- name: Store tag info
3788
shell: bash
3889
run: |
39-
docker run clux/muslrust:temp rustc --version
40-
RUST_VER="$(docker run clux/muslrust:temp rustc --version | grep -oE "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]")"
90+
mkdir -p /tmp/tags
4191
RUST_DATE="$(date +"%Y-%m-%d")"
4292
RUST_CHANNEL=nightly
43-
echo "TAG1=clux/muslrust:${RUST_CHANNEL}" >> $GITHUB_ENV
44-
echo "TAG2=clux/muslrust:${RUST_CHANNEL}-${RUST_DATE}" >> $GITHUB_ENV
45-
echo "TAG3=clux/muslrust:${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}" >> $GITHUB_ENV
93+
RUST_VER="$(docker run --platform ${{ matrix.platform }} rustmusl-temp rustc --version | grep -oE "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]")"
4694
47-
- name: Run tests
95+
echo $RUST_DATE > /tmp/tags/rust-date
96+
echo $RUST_CHANNEL > /tmp/tags/rust-channel
97+
echo $RUST_VER > /tmp/tags/rust-ver
98+
99+
- name: Tag and push
48100
shell: bash
49-
run: just test
101+
run: |
102+
RUST_DATE=$(cat /tmp/tags/rust-date)
103+
RUST_CHANNEL=$(cat /tmp/tags/rust-channel)
104+
RUST_VER=$(cat /tmp/tags/rust-ver)
105+
106+
TAG_NAME="${{ matrix.arch }}-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}"
50107
51-
- name: Push image under computed tags
52-
uses: docker/build-push-action@v2
108+
docker tag rustmusl-temp ${{ env.REGISTRY_IMAGE }}:$TAG_NAME
109+
docker push ${{ env.REGISTRY_IMAGE }}:$TAG_NAME
110+
111+
- name: Upload tags
112+
uses: actions/upload-artifact@v4
53113
with:
54-
context: .
55-
build-args: |
56-
CHANNEL=nightly
57-
push: ${{ github.event_name != 'pull_request' }}
58-
tags: clux/muslrust:latest,${{ env.TAG1 }},${{ env.TAG2 }},${{ env.TAG3 }}
114+
name: tags
115+
path: /tmp/tags
116+
if-no-files-found: error
117+
retention-days: 1
118+
overwrite: true
119+
120+
merge:
121+
runs-on: ubuntu-latest
122+
needs:
123+
- build
124+
steps:
125+
126+
-
127+
name: Download tags
128+
uses: actions/download-artifact@v4
129+
with:
130+
path: /tmp/tags
131+
name: tags
132+
133+
-
134+
name: Set up Docker Buildx
135+
uses: docker/setup-buildx-action@v3
136+
-
137+
name: Docker meta
138+
id: meta
139+
uses: docker/metadata-action@v5
140+
with:
141+
images: ${{ env.REGISTRY_IMAGE }}
142+
-
143+
name: Login to Docker Hub
144+
uses: docker/login-action@v3
145+
with:
146+
username: clux
147+
password: ${{ secrets.DOCKERHUB_TOKEN }}
148+
-
149+
name: Create manifest list and push multi-platform images
150+
run: |
151+
RUST_DATE=$(cat /tmp/tags/rust-date)
152+
RUST_CHANNEL=$(cat /tmp/tags/rust-channel)
153+
RUST_VER=$(cat /tmp/tags/rust-ver)
154+
155+
for tag in latest ${RUST_CHANNEL} ${RUST_CHANNEL}-${RUST_DATE} ${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}; do
156+
docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:$tag \
157+
${{ env.REGISTRY_IMAGE }}:amd64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE} \
158+
${{ env.REGISTRY_IMAGE }}:arm64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}
159+
done
160+
161+
-
162+
name: Inspect image
163+
run: |
164+
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest

‎Dockerfile.arm64‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
FROM ubuntu:jammy
2+
LABEL maintainer="Eirik Albrigtsen <sszynrae@gmail.com>"
3+
4+
# Required packages:
5+
# - musl-dev, musl-tools - the musl toolchain
6+
# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs
7+
# - ca-certificates - openssl + curl + peer verification of downloads
8+
# - xutils-dev - for openssl makedepend
9+
# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process
10+
# - git - cargo builds in user projects
11+
# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only)
12+
# - file - needed by rustup.sh install
13+
# - automake autoconf libtool - support crates building C deps as part cargo build
14+
# NB: does not include cmake atm
15+
RUN apt-get update && apt-get install -y \
16+
musl-dev \
17+
musl-tools \
18+
file \
19+
git \
20+
openssh-client \
21+
make \
22+
cmake \
23+
g++ \
24+
curl \
25+
pkgconf \
26+
ca-certificates \
27+
xutils-dev \
28+
libssl-dev \
29+
libpq-dev \
30+
automake \
31+
autoconf \
32+
libtool \
33+
libprotobuf-dev \
34+
unzip \
35+
--no-install-recommends && \
36+
rm -rf /var/lib/apt/lists/*
37+
38+
# Install rust using rustup
39+
ARG CHANNEL
40+
ENV RUSTUP_VER="1.26.0" \
41+
RUST_ARCH="aarch64-unknown-linux-gnu" \
42+
CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
43+
44+
RUN curl "https://static.rust-lang.org/rustup/archive/${RUSTUP_VER}/${RUST_ARCH}/rustup-init" -o rustup-init && \
45+
chmod +x rustup-init && \
46+
./rustup-init -y --default-toolchain ${CHANNEL} --profile minimal --no-modify-path && \
47+
rm rustup-init && \
48+
~/.cargo/bin/rustup target add aarch64-unknown-linux-musl
49+
50+
# Allow non-root access to cargo
51+
RUN chmod a+X /root
52+
53+
# Convenience list of versions and variables for compilation later on
54+
# This helps continuing manually if anything breaks.
55+
ENV SSL_VER="1.1.1w" \
56+
CURL_VER="8.6.0" \
57+
ZLIB_VER="1.3.1" \
58+
PQ_VER="11.12" \
59+
SQLITE_VER="3450100" \
60+
PROTOBUF_VER="25.2" \
61+
CC=musl-gcc \
62+
PREFIX=/musl \
63+
PATH=/usr/local/bin:/root/.cargo/bin:$PATH \
64+
PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \
65+
LD_LIBRARY_PATH=$PREFIX
66+
67+
# Install a more recent release of protoc (protobuf-compiler in jammy is 4 years old and misses some features)
68+
RUN cd /tmp && \
69+
curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-aarch_64.zip -o protoc.zip && \
70+
unzip protoc.zip && \
71+
cp bin/protoc /usr/bin/protoc && \
72+
rm -rf *
73+
74+
# Set up a prefix for musl build libraries, make the linker's job of finding them easier
75+
# Primarily for the benefit of postgres.
76+
# Lastly, link some linux-headers for openssl 1.1 (not used herein)
77+
RUN mkdir $PREFIX && \
78+
echo "$PREFIX/lib" >> /etc/ld-musl-aarch64.path && \
79+
ln -s /usr/include/aarch64-linux-gnu/asm /usr/include/aarch64-linux-musl/asm && \
80+
ln -s /usr/include/asm-generic /usr/include/aarch64-linux-musl/asm-generic && \
81+
ln -s /usr/include/linux /usr/include/aarch64-linux-musl/linux
82+
83+
# Build zlib (used in openssl and pq)
84+
RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \
85+
cd zlib-$ZLIB_VER && \
86+
CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \
87+
make -j$(nproc) && make install && \
88+
cd .. && rm -rf zlib-$ZLIB_VER
89+
90+
# Build openssl (used in curl and pq)
91+
# Would like to use zlib here, but can't seem to get it to work properly
92+
# TODO: fix so that it works
93+
RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz && \
94+
cd openssl-$SSL_VER && \
95+
CFLAGS="-mno-outline-atomics" ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-aarch64 && \
96+
env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \
97+
make -j$(nproc) && make all install_sw && \
98+
cd .. && rm -rf openssl-$SSL_VER
99+
100+
# Build curl (needs with-zlib and all this stuff to allow https)
101+
# curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what
102+
RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \
103+
cd curl-$CURL_VER && \
104+
CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
105+
--enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \
106+
--with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \
107+
--with-openssl --without-libpsl && \
108+
make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \
109+
cd .. && rm -rf curl-$CURL_VER
110+
111+
# Build libpq
112+
RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \
113+
cd postgresql-$PQ_VER && \
114+
CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
115+
--without-readline \
116+
--with-openssl \
117+
--prefix=$PREFIX --host=x86_64-unknown-linux-musl && \
118+
cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \
119+
cd ../../bin/pg_config && make -j $(nproc) && make install && \
120+
cd .. && rm -rf postgresql-$PQ_VER
121+
122+
# Build libsqlite3 using same configuration as the alpine linux main/sqlite package
123+
RUN curl -sSL https://www.sqlite.org/2024/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \
124+
cd sqlite-autoconf-$SQLITE_VER && \
125+
CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \
126+
CC="musl-gcc -fPIC -pie" \
127+
./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --enable-dynamic-extensions --disable-shared && \
128+
make && make install && \
129+
cd .. && rm -rf sqlite-autoconf-$SQLITE_VER
130+
131+
# SSL cert directories get overridden by --prefix and --openssldir
132+
# and they do not match the typical host configurations.
133+
# The SSL_CERT_* vars fix this, but only when inside this container
134+
# musl-compiled binary must point SSL at the correct certs (muslrust/issues/5) elsewhere
135+
# Postgres bindings need vars so that diesel_codegen.so uses the GNU deps at build time
136+
# but finally links with the static libpq.a at the end.
137+
# It needs the non-musl pg_config to set this up with libpq-dev (depending on libssl-dev)
138+
# See https://github.com/sgrif/pq-sys/pull/18
139+
ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \
140+
RUSTUP_HOME=/root/.rustup \
141+
CARGO_BUILD_TARGET=aarch64-unknown-linux-musl \
142+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld -Ctarget-feature=+crt-static" \
143+
PKG_CONFIG_ALLOW_CROSS=true \
144+
PKG_CONFIG_ALL_STATIC=true \
145+
PQ_LIB_STATIC_AARCH64_UNKNOWN_LINUX_MUSL=true \
146+
PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \
147+
PG_CONFIG_AARCH64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \
148+
OPENSSL_STATIC=true \
149+
OPENSSL_DIR=$PREFIX \
150+
SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
151+
SSL_CERT_DIR=/etc/ssl/certs \
152+
LIBZ_SYS_STATIC=1 \
153+
DEBIAN_FRONTEND=noninteractive \
154+
TZ=Etc/UTC
155+
156+
# Allow ditching the -w /volume flag to docker run
157+
WORKDIR /volume

‎Dockerfile.test-runner‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM ubuntu:jammy
2+
3+
RUN apt-get update && apt-get install -y \
4+
ca-certificates \
5+
file

‎Dockerfile‎ renamed to ‎Dockerfile.x86_64‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz &&
9494
cd openssl-$SSL_VER && \
9595
./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \
9696
env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \
97-
make -j$(nproc) && make install && \
97+
make -j$(nproc) && make all install_sw && \
9898
cd .. && rm -rf openssl-$SSL_VER
9999

100100
# Build curl (needs with-zlib and all this stuff to allow https)

‎test.sh‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,31 @@
22
set -ex
33

44
docker_build() {
5+
echo "Target dir: $TARGET_DIR"
6+
echo "Platform: $PLATFORM"
7+
58
# NB: add -vv to cargo build when debugging
69
local -r crate="$1"crate
710
docker run --rm \
811
-v "$PWD/test/${crate}:/volume" \
912
-v cargo-cache:/root/.cargo/registry \
1013
-e RUST_BACKTRACE=1 \
11-
clux/muslrust:temp \
14+
-e AR=ar \
15+
--platform $PLATFORM \
16+
rustmusl-temp \
1217
cargo build
18+
1319
cd "test/${crate}"
14-
./target/x86_64-unknown-linux-musl/debug/"${crate}"
15-
ldd "target/x86_64-unknown-linux-musl/debug/${crate}" 2>&1 | grep -qE "not a dynamic|statically linked" && \
16-
echo "${crate} is a static executable"
20+
21+
# Ideally we would use `ldd` but due to a qemu bug we can't :(
22+
# See https://github.com/multiarch/qemu-user-static/issues/172
23+
# Instead we use `file`.
24+
docker run --rm \
25+
-v "$PWD:/volume" \
26+
-e RUST_BACKTRACE=1 \
27+
--platform $PLATFORM \
28+
test-runner \
29+
bash -c "cd volume; ./target/$TARGET_DIR/debug/${crate} && file ./target/$TARGET_DIR/debug/${crate} && file /volume/target/$TARGET_DIR/debug/${crate} 2>&1 | grep -qE 'static-pie linked|statically linked' && echo ${crate} is a static executable"
1730
}
1831

1932
# Helper to check how ekidd/rust-musl-builder does it

0 commit comments

Comments
 (0)