Merge pull request #4 from fluxcd/scratch-image

hiddeco · web-flow · commit 1689bf601457 · 2021-10-07T20:53:05.000+02:00
Refactor into scratch based image
diff --git a/Dockerfile b/Dockerfile
@@ -1,56 +1,3 @@
-ARG BASE_VARIANT=bullseye
-ARG GO_VERSION=1.16.8
-ARG XX_VERSION=1.0.0-rc.2
+FROM scratch
 
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
-
-FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
-FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest
-
-FROM gostable AS go-linux
-
-FROM go-${TARGETOS} AS build-base-bullseye
-
-COPY --from=xx / /
-
-RUN apt-get update && apt-get install --no-install-recommends -y clang
-ARG CMAKE_VERSION=3.21.3
-RUN curl -sL -o cmake-linux.sh "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(xx-info march).sh" \
-    && sh cmake-linux.sh -- --skip-license --prefix=/usr \
-    && rm cmake-linux.sh
-
-FROM build-base-bullseye AS build-bullseye
-ARG TARGETPLATFORM
-RUN xx-apt install --no-install-recommends -y binutils gcc libc6-dev dpkg-dev
-
-FROM build-${BASE_VARIANT} AS build-dependencies-bullseye
-
-# Install libssh2 for $TARGETPLATFORM from "sid", as the version in "bullseye"
-# has been linked against gcrypt, which causes issues with PKCS* formats.
-# We pull (sub)dependencies from there as well, to ensure all versions are aligned,
-# and not accidentially linked to e.g. mbedTLS (which has limited support for
-# certain key formats).
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-# Ref: https://github.com/ARMmbed/mbedtls/issues/2452#issuecomment-802683144
-ARG TARGETPLATFORM
-RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list
-RUN xx-apt update \
-    && xx-apt -t sid install --no-install-recommends -y zlib1g-dev libssl-dev libssh2-1-dev
-
-FROM build-dependencies-${BASE_VARIANT} as build-libgit2-bullseye
-
-# Compile libgit2 as a dynamic build
-# We compile it ourselves to ensure they are properly linked with the above packages,
-# and to allow room for customizations (or more rapid updates than the OS).
-ARG LIBGIT2_PATH=/libgit2
-ENV LIBGIT2_PATH=${LIBGIT2_PATH}
-COPY hack/Makefile ${LIBGIT2_PATH}/Makefile
-RUN set -e; \
-    echo "/usr/lib/$(xx-info triple)" > ${LIBGIT2_PATH}/INSTALL_LIBDIR \
-    && INSTALL_LIBDIR=$(cat ${LIBGIT2_PATH}/INSTALL_LIBDIR) \
-    FLAGS=$(xx-clang --print-cmake-defines) \
-    make -C ${LIBGIT2_PATH} \
-    && xx-verify $(cat ${LIBGIT2_PATH}/INSTALL_LIBDIR)/libgit2.so \
-    && mkdir -p ${LIBGIT2_PATH}/lib/ \
-    && cp -d $(cat ${LIBGIT2_PATH}/INSTALL_LIBDIR)/libgit2.so* ${LIBGIT2_PATH}/lib/
+COPY hack/Makefile Makefile
diff --git a/Dockerfile.test b/Dockerfile.test
@@ -1,11 +1,30 @@
-# This Docker image can be used to confirm the sets of dependencies can be used together with git2go.
-# Note: this just confirms the C library wiring is working, and no implementation details.
-# It does for example not tell you if ED25519 is supported by the underlying set of dependencies, etc.
-ARG IMG
-ARG TAG
-# NB: because the `Dockerfile` performs a build on the $BUILDPLATFORM and then later pushes it as
-# if it were a $TARGETPLATFORM image, we need to inverse this by NOT using --platform.
-FROM $IMG:$TAG as source
+# This Dockerfile tests the hack/Makefile output against git2go.
+ARG BASE_VARIANT=bullseye
+ARG GO_VERSION=1.16.8
+ARG XX_VERSION=1.0.0-rc.2
+
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+
+FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+
+FROM gostable AS go-linux
+
+FROM go-${TARGETOS} AS build-base-bullseye
+
+COPY --from=xx / /
+COPY hack/Makefile /libgit2/Makefile
+
+RUN make -C /libgit2/ cmake
+
+ARG TARGETPLATFORM
+RUN make -C /libgit2/ dependencies
+
+FROM build-base-${BASE_VARIANT} as libgit2-bullseye
+
+ARG TARGETPLATFORM
+RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2/ libgit2
+
+FROM libgit2-${BASE_VARIANT} as test-bullseye
 
 # Cache clone
 ARG GIT2GO_TAG
@@ -17,17 +36,10 @@ RUN git config --global advice.detachedHead false \
 # Set workdir
 WORKDIR /git2go
 
-FROM source as test
-
 # Run tests
 ARG TARGETPLATFORM
 ARG CACHE_BUST
 RUN set -eux; \
     echo "=> Dynamic test at $CACHE_BUST for $TARGETPLATFORM" \
     && CGO_ENABLED=1 xx-go run script/check-MakeGitError-thread-lock.go \
     && CGO_ENABLED=1 xx-go test ./...
-
-#RUN set -eux; \
-#    echo "=> Static test at $CACHE_BUST for $TARGETPLATFORM" \
-#    && CGO_ENABLED=1 xx-go run -tags "static,system_libgit2" script/check-MakeGitError-thread-lock.go \
-#    && CGO_ENABLED=1 xx-go test -tags "static,system_libgit2" ./...
diff --git a/README.md b/README.md
@@ -1,8 +1,7 @@
 # golang-with-libgit2
 
-This repository contains a `Dockerfile` which makes use of [`tonistiigi/xx`][xx] to produce a [Go container image][]
-with a dynamic set of [libgit2][] dependencies. The image can be used to build **AMD64, ARM64 and ARMv7** binaries of Go
-projects that depend on [git2go][].
+This repository contains a `Dockerfile` which `Makefile` content can be used to build the [libgit2][] dependency
+chain for **AMD64, ARM64 and ARMv7** binaries of Go projects that depend on [git2go][]. 
 
 ### :warning: **Public usage discouraged**
 
@@ -26,8 +25,8 @@ that producing a set of dependencies that work for all end-users using OS packag
 - [fluxcd/image-automation-controller#186](https://github.com/fluxcd/image-automation-controller/issues/186)
 - [fluxcd/source-controller#439](https://github.com/fluxcd/source-controller/issues/439)
 
-This image is an attempt to solve (most of) these issues, by compiling `libgit2` ourselves, linking it with
-the required dependencies at specific versions and with specific configuration, and linker options,
+This image is an attempt to solve (most of) these issues, by providing the configuration to compile `libgit2` ourselves,
+linking it with the required dependencies at specific versions and with specific configuration, and linker options,
 while testing these against the git2go code before releasing the image.
 
 ### List of known issues
@@ -45,18 +44,9 @@ while testing these against the git2go code before releasing the image.
 
 ## Usage
 
-To make use of the image published by the `Dockerfile`, use it as a base image for your Go build. In your application
-container, ensure the [runtime dependencies](#Runtime-dependencies) are present, and copy over the `libgit2` shared
-libraries from `$LIBGIT2_PATH/lib/*` (default `/libgit2/lib`).
-
-### libgit2
-
-The `libgit2` library is installed to the expected path for the `$TARGETPLATFORM`, and building a dynamically linked Go
-binary should be possible by just running `xx-go build`. For the application image, a copy of the `.so*` files are
-available in `$LIBGIT2_PATH/lib`.
-
-In cases where you need to determine the architecture based library installation path without making use of `xx[-info]`,
-the destination path is written to `$LIBGIT2_PATH/INSTALL_LIBDIR`.
+To make use of the image published by the `Dockerfile`, copy over the `Makefile` from the image as a prerequisite to
+your Go build. Once copied over to the image, the set of targets can be used to compile `libgit2` for the
+`$BUILDPLATFORM` and/or `$TARGETPLATFORM` (see [example](#Dockerfile-example)).
 
 ### Runtime dependencies
 
@@ -74,7 +64,25 @@ The following dependencies should be present in the image running the applicatio
 ### `Dockerfile` example
 
 ```Dockerfile
-FROM hiddeco/golang-with-libgit2 AS build
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+FROM fluxcd/golang-with-libgit2 as libgit2
+
+FROM --platform=$BUILDPLATFORM golang:1.16.8-bullseye as build
+
+# Copy the build utiltiies
+COPY --from=xx / /
+COPY --from=libgit2 /Makefile /libgit2/
+
+# Install the libgit2 build dependencies
+RUN make -C /libgit2 cmake
+
+ARG TARGETPLATFORM
+RUN make -C /libgit2 dependencies
+
+# Compile and install libgit2
+RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 \
+    && mkdir -p /libgit2/lib/ \
+    && cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/
 
 # Configure workspace
 WORKDIR /workspace
@@ -119,13 +127,18 @@ ENTRYPOINT [ "app" ]
 
 ## Contributing
 
-### Updating the Go version
+### Updating the `libgit2` version
 
-In the `Dockerfile`, update the default value of the `GO_VERSION` to the new target version.
+Change the default value of `LIBGIT2_VERSION` in `hack/Makefile`. If applicable, change the `GIT2GO_TAG` in the
+`Makefile` in the repository root as well to test against another version of [git2go][].
+
+### Updating the test Go version
 
-### Updating the base image variant
+In the `Dockerfile.test`, update the default value of the `GO_VERSION` to the new target version.
 
-In the `Dockerfile`, update the `BASE_VARIANT` to the new target base variant. Then, ensure all build stages making use
+### Updating the test base image variant
+
+In the `Dockerfile.test`, update the `BASE_VARIANT` to the new target base variant. Then, ensure all build stages making use
 of (or depending on) the base `${BASE_VARIANT}`, use it in their `AS` stage defined for the new variant.
 
 For example:
@@ -140,23 +153,16 @@ FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as go-awesom
 FROM go-${BASE_VARIANT} AS build-dependencies-awesome-os
 ```
 
-### Updating the `libgit2` version
-
-Change the default value of `LIBGIT2_VERSION` in `hack/Makefile`. If applicable, change the `GIT2GO_TAG` in the
-`Makefile` in the repository root as well to test against another version of [git2go][].
-
 ### Releasing a new image
 
 For the `main` branch, images are pushed automatically to a tag matching the branch name, and a tag in the format of
-`sha-<Git sha>`. In addition, images are created for new tags, with as preferred format:
-`<Go SemVer>-<image variant>-libgit2-<libgit2 SemVer>`.
+`sha-<Git sha>`. In addition, images are created for new tags, with as preferred format: `libgit2-<libgit2 SemVer>`.
 
-For example, `1.16.8-bullseye-libgit2-1.1.1` for an image with
-**Go 1.16.8** based on **Debian bullseye** with **libgit2 1.1.1** included.
+For example, `libgit2-1.1.1` for an image with **libgit2 1.1.1** included.
 
-In case changes happen to the `Dockerfile` while the `Go` or `libgit2` versions do not change, sequential tags should
-be suffixed with `-<seq num in range>`. For example, `1.16.8-bullseye-libgit2-1.1.1-2` for the **third** container image
-with these versions.
+In case changes happen to the `Dockerfile` while the `libgit2` version does not change, sequential tags should
+be suffixed with `-<seq num in range>`. For example, `libgit2-1.1.1-2` for the **third** container image
+with the same version.
 
 [xx]: https://github.com/tonistiigi/xx
 [Go container image]: https://hub.docker.com/_/golang
diff --git a/hack/Makefile b/hack/Makefile
@@ -1,61 +1,99 @@
-BUILD_TYPE ?= "RelWithDebInfo"
-FLAGS ?=
+# This determines if we are building while making use of xx. Ensuring
+# the package directives target the right manager and inject (OS) vendor
+# specific configurations.
+IS_XX := $(shell command -v xx-info 2> /dev/null)
+ifdef IS_XX
+XX_VENDOR ?= $(shell xx-info vendor)
+endif
 
+# Libgit2 build configuration flags.
 INSTALL_PREFIX ?= /usr
+ifndef IS_XX
 INSTALL_LIBDIR ?= $(INSTALL_PREFIX)/lib
+else
+INSTALL_LIBDIR ?= $(INSTALL_PREFIX)/lib/$(shell xx-info triple)
+endif
+BUILD_TYPE ?= "RelWithDebInfo"
+FLAGS ?=
 USE_HTTPS ?= OpenSSL
 USE_SSH ?= ON
 
+# Cmake version to be installed.
+CMAKE_VERSION ?= 3.21.3
+
+# Libgit2 version to be compiled and installed.
 LIBGIT2_VERSION ?= 1.1.1
-# Overwritten to a specific version by default for git2go support
+# In some scenarios libgit2 needs to be checked out to a specific commit.
+# This takes presidence over LIBGIT_VERSION if defined.
 # Ref: https://github.com/libgit2/git2go/issues/834
 LIBGIT2_REVISION ?=
 
-ifeq ($(LIBGIT2_REVISION),)
+# Set the download URL based on the above information.
+ifeq (,$(LIBGIT2_REVISION))
 LIBGIT2_DOWNLOAD_URL ?= https://github.com/libgit2/libgit2/archive/refs/tags/v$(LIBGIT2_VERSION).tar.gz
 else
 LIBGIT2_DOWNLOAD_URL ?= https://github.com/libgit2/libgit2/archive/$(LIBGIT2_REVISION).tar.gz
 endif
 
-LIBGIT2_LIB := $(INSTALL_LIBDIR)/libgit2.so.$(LIBGIT2_VERSION)
-
-PKG_CONFIG_PATH ?=
-# Detect Darwin (MacOS) to help the user a bit configuring OpenSSL,
-# or at least in pointing out what steps should be taken if it can't
-# magically figure it out by itself. Because we too, are nice people.
-ifeq ($(shell uname -s),Darwin)
-	LIBGIT2_LIB := $(INSTALL_LIBDIR)/libgit2.$(LIBGIT2_VERSION).dylib
+# OS specific expected lib output.
+LIBGIT2 := $(INSTALL_LIBDIR)/libgit2.so.$(LIBGIT2_VERSION)
+ifeq (Darwin,$(shell uname -s))
+	LIBGIT2 := $(INSTALL_LIBDIR)/libgit2.$(LIBGIT2_VERSION).dylib
 	HAS_BREW := $(shell brew --version 2>/dev/null)
+endif
+
+cmake:
+ifeq (debian,$(XX_VENDOR))
+cmake:
+	apt-get update && apt-get install --no-install-recommends -y clang cmake
+endif
+.PHONY: cmake
+
+base:
+ifeq (debian,$(XX_VENDOR))
+base:
+	xx-apt update && xx-apt install --no-install-recommends -y binutils gcc libc6-dev dpkg-dev
+endif
+.PHONY: base
+
+dependencies: base
+ifeq (debian,$(XX_VENDOR))
+# Install libssh2 for $TARGETPLATFORM from "sid", as the version in "bullseye"
+# has been linked against gcrypt, which causes issues with PKCS* formats.
+# We pull (sub)dependencies from there as well, to ensure all versions are aligned,
+# and not accidentially linked to e.g. mbedTLS (which has limited support for
+# certain key formats).
+# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
+# Ref: https://github.com/ARMmbed/mbedtls/issues/2452#issuecomment-802683144
+dependencies:
+	set -e; \
+	echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list.d/sid.list \
+	&& echo "deb-src http://deb.debian.org/debian sid main" /etc/apt/sources.list.d/sid.list \
+	&& xx-apt update \
+	&& xx-apt -t sid install --no-install-recommends -y zlib1g-dev libssl-dev libssh2-1-dev
+endif
+.PHONY: dependencies
+
+libgit2: $(LIBGIT2)
+.PHONY: libgit2
+
 ifdef HAS_BREW
 HAS_OPENSSL := $(shell brew --prefix openssl@1.1)
 # NB: the OPENSSL_LDFLAGS ensures the path is included in the libgit2.pc
-# file. As standard brew installation doesn't appear to be system wide
-# on most macOS instances, and you thus have to tell explicitly where
+# file. As a standard brew installation doesn't appear to be system wide
+# on most macOS instances, and we thus have to explicitly tell where
 # it can be found.
 ifdef HAS_OPENSSL
 	PKG_CONFIG_PATH := $(PKG_CONFIG_PATH):$(HAS_OPENSSL)/lib/pkgconfig
-	OS_FLAGS += -DOPENSSL_LDFLAGS:STRING=-L $(HAS_OPENSSL)/lib
-else
-	$(warning "Failed to detect openssl@1.1 installation with brew. It can be installed with 'brew install openssl@1.1',")
-	$(warning "or an alternative location can be provided using the PKG_CONFIG_PATH flag, for example:")
-	$(warning "'PKG_CONFIG_PATH=/usr/local/lib/pkgconfig make'.")
+	FLAGS += -DOPENSSL_LDFLAGS:STRING=-L $(HAS_OPENSSL)/lib
 endif
 HAS_LIBSSH2 := $(shell brew --prefix libssh2)
 ifdef HAS_LIBSSH2
 	PKG_CONFIG_PATH := $(PKG_CONFIG_PATH):$(HAS_LIBSSH2)/lib/pkgconfig
-else
-	$(warning "Failed to detect libssh2 installation with brew. It can be installed with 'brew install libssh2',")
-	$(warning "or an alternative location can be provided using the PKG_CONFIG_PATH flag, for example:")
-	$(warning "'PKG_CONFIG_PATH=/usr/local/lib/pkgconfig make'.")
-endif
-else
-	$(warning "Failed to detect brew installation, and therefore unable to automatically determine lib paths.")
-	$(warning "The location of openssl and libssh2 can be provided using the PKG_CONFIG_PATH flag, for example:")
-	$(warning "'PKG_CONFIG_PATH=/usr/local/lib/pkgconfig make'.")
 endif
 endif
 
-$(LIBGIT2_LIB):
+$(LIBGIT2):
 	set -e; \
 	SETUP_LIBGIT2_TMP_DIR=$$(mktemp -d) \
 	&& curl -L $(LIBGIT2_DOWNLOAD_URL) -o $$SETUP_LIBGIT2_TMP_DIR/archive.tar.gz \
@@ -76,25 +114,6 @@ $(LIBGIT2_LIB):
 		-DREGEX_BACKEND:STRING=builtin \
 		-DUSE_HTTPS:STRING=$(USE_HTTPS) \
 		-DUSE_SSH:BOOL=$(USE_SSH) \
-		$(OS_FLAGS) \
 		$(FLAGS) \
 	&& cmake --build $$SETUP_LIBGIT2_TMP_DIR/build --target install \
 	&& rm -rf $$SETUP_LIBGIT2_TMP_DIR
-
-#    \ && cmake -S $tmp/src -B $tmp/build $(xx-clang --print-cmake-defines) \
-#      -DCMAKE_BUILD_TYPE:STRING=$LIBGIT2_BUILD_TYPE \
-#      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \
-#      -DCMAKE_C_FLAGS=-fPIC \
-#      -DCMAKE_CXX_FLAGS=-fPIC \
-#      -DDEPRECATE_HARD=ON \
-#      -DCMAKE_INSTALL_PREFIX:PATH=/usr \
-#      -DCMAKE_INSTALL_LIBDIR:PATH=/usr/lib/$(xx-info triple) \
-#      -DBUILD_CLAR:BOOL:BOOL=OFF \
-#      -DTHREADSAFE:BOOL=ON \
-#      -DBUILD_SHARED_LIBS=OFF \
-#      -DUSE_BUNDLED_ZLIB:BOOL=OFF \
-#      -DUSE_HTTP_PARSER:STRING=builtin  \
-#      -DREGEX_BACKEND:STRING=builtin \
-#      -DUSE_HTTPS:STRING=OpenSSL \
-#      -DUSE_SSH:BOOL=ON \
-#    && cmake --build $tmp/build --target install \
