Optimise cross compilation time

Signed-off-by: Paulo Gomes <paulo.gomes@weave.works>

Author: Paulo Gomes

Dockerfile

@@ -37,6 +37,23 @@ COPY go.sum go.sum
 # Cache modules
 RUN go mod download
 
+# The musl-tool-chain layer is an adhoc solution
+# for the problem in which xx gets confused during compilation
+# and a) looks for gold linker and then b) cannot find musl's dynamic linker.
+FROM --platform=$BUILDPLATFORM alpine as musl-tool-chain
+
+COPY --from=xx / /
+
+RUN apk add bash curl tar
+
+WORKDIR /workspace
+COPY hack/download-musl.sh .
+
+ARG TARGETPLATFORM
+ARG TARGETARCH
+RUN ROOT_DIR="$(pwd)" TARGET_ARCH="$(xx-info alpine-arch)" ENV_FILE=true \
+        ./download-musl.sh
+
 # Build stage install per target platform
 # dependency and effectively cross compile the application.
 FROM build-go-mod as build
@@ -47,10 +64,7 @@ COPY --from=libgit2-libs /usr/local/ /usr/local/
 
 # Some dependencies have to installed 
 # for the target platform: https://github.com/tonistiigi/xx#go--cgo
-RUN xx-apk add --no-cache \
-        musl-dev gcc lld binutils-gold
-
-RUN xx-apk add --no-cache musl-utils
+RUN xx-apk add musl-dev gcc lld
 
 WORKDIR /workspace
 
@@ -60,12 +74,19 @@ COPY controllers/ controllers/
 COPY pkg/ pkg/
 COPY internal/ internal/
 
+COPY --from=musl-tool-chain /workspace/build /workspace/build
+
+ARG TARGETPLATFORM
+ARG TARGETARCH
 ENV CGO_ENABLED=1
-RUN export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \
+
+# Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain.
+RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \
+    export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \
     export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \
     export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \
     export CGO_LDFLAGS="${FLAGS} -static" && \
-    xx-go build  \
+    GOARCH=$TARGETARCH go build  \
         -ldflags "-s -w" \
         -tags 'netgo,osusergo,static_build' \
         -o /source-controller -trimpath main.go;

hack/download-musl.sh

@@ -6,26 +6,61 @@ MUSL_X86_64_FILENAME=x86_64-linux-musl-native.tgz
 MUSL_X86_64_SHA512=44d441ad9aa11a06feddf3daa4c9f53ad7d9ca37af1f5a61379aca07793703d179410cea723c1b7fca94c4de19a321228bdb3656bc5cbdb5e3bea8e2d6dac6c7
 MUSL_AARCH64_FILENAME=aarch64-linux-musl-native.tgz
 MUSL_AARCH64_SHA512=16d544e09845c9dbba50f29e0cb04dd661e17eb63c56acad6a67fd2a78aa7596b792477c7177d3cd56d408a27dc291a90507df882f2b099c0f25511ce08fd3b5
+MUSL_XX86_64_FILENAME=x86_64-linux-musl-cross.tgz
+MUSL_XX86_64_SHA512=52abd1a56e670952116e35d1a62e048a9b6160471d988e16fa0e1611923dd108a581d2e00874af5eb04e4968b1ba32e0eb449a1f15c3e4d5240ebe09caf5a9f3
+MUSL_XAARCH64_FILENAME=aarch64-linux-musl-cross.tgz
+MUSL_XAARCH64_SHA512=8695ff86979cdf30fbbcd33061711f5b1ebc3c48a87822b9ca56cde6d3a22abd4dab30fdcd1789ac27c6febbaeb9e5bde59d79d66552fae53d54cc1377a19272
+MUSL_XARMV7_FILENAME=armv7l-linux-musleabihf-cross.tgz
+MUSL_XARMV7_SHA512=1bb399a61da425faac521df9b8d303e60ad101f6c7827469e0b4bc685ce1f3dedc606ac7b1e8e34d79f762a3bfe3e8ab479a97e97d9f36fbd9fc5dc9d7ed6fd1
 
-MUSL_FILENAME="${MUSL_X86_64_FILENAME}"
-MUSL_SHA512="${MUSL_X86_64_SHA512}"
-if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-    MUSL_FILENAME="${MUSL_AARCH64_FILENAME}"
-    MUSL_SHA512="${MUSL_AARCH64_SHA512}"
-fi
+TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
+ENV_FILE="${ENV_FILE:-false}"
 
-MUSL_AARCH64_URL="https://more.musl.cc/11.2.1/x86_64-linux-musl/${MUSL_FILENAME}"
+MUSL_FILENAME=""
+MUSL_SHA512=""
 
-ROOT_DIR="$(git rev-parse --show-toplevel)"
+ROOT_DIR="${ROOT_DIR:-$(git rev-parse --show-toplevel)}"
 MUSL_DIR="${ROOT_DIR}/build/musl"
 
+
+if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then
+    MUSL_FILENAME="${MUSL_X86_64_FILENAME}"
+    MUSL_SHA512="${MUSL_X86_64_SHA512}"
+    MUSL_PREFIX=$(xx-info alpine-arch)-linux-musl-native/bin/$(xx-info alpine-arch)-linux-musl
+    if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
+        MUSL_FILENAME="${MUSL_AARCH64_FILENAME}"
+        MUSL_SHA512="${MUSL_AARCH64_SHA512}"
+    fi
+else
+    MUSL_FILENAME="${MUSL_XX86_64_FILENAME}"
+    MUSL_SHA512="${MUSL_XX86_64_SHA512}"
+    MUSL_PREFIX=$(xx-info alpine-arch)-linux-musl-cross/bin/$(xx-info alpine-arch)-linux-musl
+    if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
+        MUSL_FILENAME="${MUSL_XAARCH64_FILENAME}"
+        MUSL_SHA512="${MUSL_XAARCH64_SHA512}"
+    elif [ "${TARGET_ARCH}" = "arm" ] || [ "${TARGET_ARCH}" = "armv7" ]; then
+        MUSL_FILENAME="${MUSL_XARMV7_FILENAME}"
+        MUSL_SHA512="${MUSL_XARMV7_SHA512}"
+        MUSL_PREFIX=armv7l-linux-musleabihf-cross/bin/armv7l-linux-musleabihf
+    fi
+fi
+
+mkdir -p "${MUSL_DIR}"
+
+if "${ENV_FILE}"; then
+    cat<<EOF > "${MUSL_DIR}/${TARGET_ARCH}.env"
+CC="$(pwd)/build/musl/${MUSL_PREFIX}-gcc"
+CXX="$(pwd)/build/musl/${MUSL_PREFIX}-g++"
+AR="$(pwd)/build/musl/${MUSL_PREFIX}-ar"
+EOF
+fi
+
+MUSL_AARCH64_URL="https://more.musl.cc/11.2.1/x86_64-linux-musl/${MUSL_FILENAME}"
+
 if [ ! -f "${MUSL_DIR}/bin" ]; then
     TARGET_FILE="${MUSL_DIR}/${MUSL_FILENAME}"
-    mkdir -p "${MUSL_DIR}"
-
-    echo "${MUSL_SHA512}  ${TARGET_FILE}"
     curl -o "${TARGET_FILE}" -LO "${MUSL_AARCH64_URL}"
-    if ! echo "${MUSL_SHA512}  ${TARGET_FILE}" | sha512sum --check; then
+    if ! echo "${MUSL_SHA512}  ${TARGET_FILE}" | sha512sum; then
         echo "Checksum failed for ${MUSL_FILENAME}."
         rm -rf "${MUSL_DIR}"
         exit 1