introduce zig cc cross-compilation setup for GitHub Actions and local development

Makefile

@@ -429,6 +429,36 @@ refresh-lock-files:
 scan-image-vulnerabilities:
 	python ci/security-scan/quay_security_analysis.py
 
+ARCH := $(shell uname -m)
+ifeq ($(ARCH),amd64)
+    ARCH := x86_64
+else ifeq ($(ARCH),arm64)
+    ARCH := aarch64
+endif
+
+ZIG_VERSION := 0.15.2
+ZIG_BINARY := zig-$(ZIG_VERSION)
+
+bin/zig-$(ZIG_VERSION):
+	@echo "Installing Zig $(ZIG_VERSION)..."
+	TMPDIR=$(shell mktemp -d)
+	wget --progress=dot:giga https://ziglang.org/download/$(ZIG_VERSION)/zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz
+	tar -xJf zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz -C $$TMPDIR --strip-components=1
+	rm -rf zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz
+
+	mv $$TMPDIR bin/zig-$(ZIG_VERSION)
+	@echo "Zig installed as bin/zig-$(ZIG_VERSION)"
+
+# This should be .PHONY because it's an alias/action
+.PHONY: install-zig
+install-zig: bin/zig-$(ZIG_VERSION)
+	@echo "Zig is ready to use!"
+
+# Another .PHONY target for cleanup
+.PHONY: clean-zig
+clean-zig:
+	rm -rf bin/zig-$(ZIG_VERSION)
+
 # This is used primarily for gen_gha_matrix_jobs.py to we know the set of all possible images we may want to build
 .PHONY: all-images
 ifeq ($(RELEASE_PYTHON_VERSION), 3.12)

runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu

@@ -13,6 +13,8 @@ WORKDIR /opt/app-root/bin
 # OS Packages needs to be installed as root
 USER 0
 
+RUN ls /mnt && true
+
 ### BEGIN upgrade first to avoid fixable vulnerabilities
 # If we have a Red Hat subscription prepared, refresh it
 RUN /bin/bash <<'EOF'
@@ -39,7 +41,7 @@ ARCH=$(uname -m)
 echo "Detected architecture: $ARCH"
 PACKAGES="perl mesa-libGL skopeo"
 if [ "$ARCH" = "s390x" ] || [ "$ARCH" = "ppc64le" ]; then
-    PACKAGES="$PACKAGES gcc g++ make openssl-devel autoconf automake libtool cmake"
+    PACKAGES="$PACKAGES gcc g++ ninja-build openssl-devel autoconf automake libtool cmake"
 fi
 dnf install -y --setopt=keepcache=1 $PACKAGES
 EOF

scripts/sandbox.py

@@ -47,7 +47,39 @@ def main() -> int:
 
     with tempfile.TemporaryDirectory(delete=True) as tmpdir:
         setup_sandbox(prereqs, pathlib.Path(tmpdir))
-        command = [arg if arg != "{};" else tmpdir for arg in args.remaining[1:]]
+        additional_arguments = [
+            # Mount the zigcc utility
+            f"--volume={os.getcwd()}/bin/zig-0.15.2:/mnt",
+            f"--env=ZIGCC_ARCH={args.platform.split('/')[1]}",
+            "--unsetenv=ZIGCC_ARCH",
+            # CMake heeds these
+            "--env=CC=/mnt/cc",
+            "--env=CXX=/mnt/c++",
+            "--unsetenv=CC",
+            "--unsetenv=CXX",
+            # CMake ignores these
+            "--env=AR=/mnt/ar",
+            "--env=RANLIB=/mnt/ranlib",
+            "--env=STRIP=/mnt/strip",
+            "--unsetenv=AR",
+            "--unsetenv=RANLIB",
+            "--unsetenv=STRIP",
+            # Workaround for a s390x compilation issue
+            # Codeserver: SPDLOG_CONSTEXPR_FUNC is to work around a consteval issue with zig c++
+            #  ../deps/spdlog/include/spdlog/details/fmt_helper.h:105:54: error: call to consteval function 'fmt::basic_format_string<...>' is not a constant expression
+            #  Clang (via Zig) is stricter about consteval requirements than GCC
+            #  The format string "{:02}" cannot be evaluated as a constant expression in this context
+            "--env=CXXFLAGS=-Dundefined=64 -DFMT_CONSTEVAL= -DSPDLOG_CONSTEXPR_FUNC=",
+            "--unsetenv=CXXFLAGS",
+
+            tmpdir,
+        ]
+        command = []
+        for arg in args.remaining[1:]:
+            if arg == "{};":
+                command.extend(additional_arguments)
+            else:
+                command.append(arg)
         print(f"running {command=}")
         try:
             subprocess.check_call(command)
@@ -56,13 +88,77 @@ def main() -> int:
             return err.returncode
     return 0
 
+"""
+Downloading jedi
+  × Failed to build `pyzmq==27.1.0`
+  ├─▶ The build backend returned an error
+  ╰─▶ Call to `scikit_build_core.build.build_wheel` failed (exit status: 1)
+      [stdout]
+      *** scikit-build-core 0.11.6 using CMake 3.26.5 (wheel)
+      *** Configuring CMake...
+      loading initial cache file /tmp/tmpf9bnfh5o/build/CMakeInit.txt
+      -- Configuring incomplete, errors occurred!
+      [stderr]
+      CMake Error at /usr/share/cmake/Modules/CMakeDetermineCCompiler.cmake:49
+      (message):
+        Could not find compiler set in environment variable CC:
+        /mnt/zig-0.15.1/zig cc -target s390x-linux-gnu.
+      Call Stack (most recent call first):
+        CMakeLists.txt:2 (project)
+"""
+
+"""
+creating build/temp.linux-s390x-cpython-312/psutil/arch/linux
+      /mnt/zig cc -target s390x-linux-gnu -fno-strict-overflow
+      -Wsign-compare -DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG
+      -O2 -fexceptions -g -grecord-gcc-switches -pipe
+      -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2
+      -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong
+      -m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables
+      -fstack-clash-protection -O2 -fexceptions -g -grecord-gcc-switches
+      -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2
+      -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong
+      -m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables
+      -fstack-clash-protection -O2 -fexceptions -g -grecord-gcc-switches
+      -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2
+      -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong
+      -m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables
+      -fstack-clash-protection -fPIC -DPSUTIL_POSIX=1 -DPSUTIL_SIZEOF_PID_T=4
+      -DPSUTIL_VERSION=700 -DPy_LIMITED_API=0x03060000
+      -DPSUTIL_LINUX=1 -I/tmp/.tmpWlL4ZP/builds-v0/.tmpOwAhw2/include
+      -I/usr/include/python3.12 -c psutil/_psutil_common.c -o
+      build/temp.linux-s390x-cpython-312/psutil/_psutil_common.o
+      [stderr]
+      /tmp/.tmpWlL4ZP/builds-v0/.tmpOwAhw2/lib64/python3.12/site-packages/setuptools/dist.py:759:
+      SetuptoolsDeprecationWarning: License classifiers are deprecated.
+      !!
+
+      ********************************************************************************
+              Please consider removing the following classifiers in favor of a
+      SPDX license expression:
+              License :: OSI Approved :: BSD License
+              See
+      https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license
+      for details.
+
+      ********************************************************************************
+      !!
+        self._finalize_license_expression()
+      error: unsupported preprocessor arg: -D_FORTIFY_SOURCE
+"""
 
 def buildinputs(
         dockerfile: pathlib.Path | str,
         platform: Literal["linux/amd64", "linux/arm64", "linux/s390x", "linux/ppc64le"] = "linux/amd64"
 ) -> list[pathlib.Path]:
     if not (ROOT_DIR / "bin/buildinputs").exists():
         subprocess.check_call([MAKE, "bin/buildinputs"], cwd=ROOT_DIR)
+    if not (ROOT_DIR / "bin/zig-0.15.2").exists():
+        subprocess.check_call([MAKE, "bin/zig-0.15.2"], cwd=ROOT_DIR)
+    if not (ROOT_DIR / "bin/zig-0.15.2/zigcc").exists():
+        subprocess.check_call([MAKE, "build"], cwd=ROOT_DIR / "scripts/zigcc")
+    for alias in ["cc", "c++", "ar", "llvm-ar", "ranlib", "llvm-ranlib", "strip", "llvm-strip"]:
+        shutil.copy(ROOT_DIR / "scripts/zigcc/bin/zigcc", ROOT_DIR / "bin/zig-0.15.2" / alias)
     stdout = subprocess.check_output([ROOT_DIR / "bin/buildinputs", str(dockerfile)],
                                      text=True, cwd=ROOT_DIR,
                                      env={"TARGETPLATFORM": platform, **os.environ})

scripts/zigcc/Makefile

@@ -0,0 +1,21 @@
+.PHONY: build test clean
+
+build: bin/zigcc
+
+# always build for linux and the machine's native architecture
+bin/zigcc: *.go go.mod
+	GOOS=linux go build -o $@ -ldflags="-s -w" -v ./...
+
+test:
+	go test -v ./...
+
+fmt:
+	go fmt ./...
+
+vet:
+	go vet ./...
+
+clean:
+	go clean
+	rm -f bin/*
+	rmdir bin

scripts/zigcc/README.md

@@ -0,0 +1,68 @@
+# zigcc
+
+_Launcher for `zig cc`, `zig c++`, and related subcommands for more efficient cross-compilation_
+
+This script helps by using a native compiler binary to compile for the target architecture.
+This means that the compiler then is running at the native speed.
+
+## Background
+
+Cross-compilation means compiling for a different architecture than the one on which the compiler is running.
+Specifically, it is when you're creating a workbench container image for say x86_64 on an arm64 machine such as an M-series MacBook.
+Everything that's running in the container will run slower, because it has to run under qemu.
+
+This slowdown is especially noticeable when compiling C/C++ code for IBM Power and Z, such as Python extension modules that don't have precompiled binaries for these architectures on PyPI.
+
+## Usage
+
+```commandline
+gmake codeserver-ubi9-python-3.12 BUILD_ARCH=linux/s390x CONTAINER_BUILD_CACHE_ARGS=
+```
+
+This is about 50% faster than cross-compiling through `qemu-s390x-static` or `qemu-ppc64le-static`.
+
+## Cross-compilation overview
+
+### Qemu-user-static
+
+Docker/Podman can perform cross-compilation using `qemu-user-static`.
+The idea is to install the various `qemu-user` binaries as interpreters for foreign architecture binaries.
+Launching such binary will then automatically run it under qemu interpreter.
+
+Docker is uniquely suitable to run binaries like this, because container images bring all dependencies with them.
+
+### Traditional cross-compilation
+
+For CMake, I can imagine an approach which involves installing a cross compiler and mounting arm64 docker image to provide arm64 environment with libraries.
+<https://cmake.org/cmake/help/book/mastering-cmake/chapter/Cross%20Compiling%20With%20CMake.html>
+
+### Zig
+
+https://zig.guide/working-with-c/zig-cc/
+
+The `zig cc` command bundles clang in a way that simplifies its usage for cross compilation,
+<https://zig.news/kristoff/cross-compile-a-c-c-project-with-zig-3599>
+
+#### Wrapper (zigcc.go)
+
+We need a wrapper so that we can transform CLI arguments to work with `zig cc`.
+
+The main problem is the `-Wl,D_FORTIFY_SOURCE=2` flag, because zig has limited handling for -Wl, and does not do -Wl,D correctly.
+
+The wrapper should be written in a low-overhead language, like Go, or possibly Bash, certainly not Python.
+The lower the overhead of the wrapper, the better, since the compiler is invoked many times during a typical build.
+
+### Debugging
+
+To observe the effect of the wrapper, we can use `execsnoop` from `bcc-tools` to monitor the compiler invocations during a container build.
+
+```commandline
+$ podman machine ssh
+# bootc usr-overlay
+# dnf install bcc-tools
+# /usr/share/bcc/tools/execsnoop
+```
+
+## Credits
+
+This is inspired by <https://github.com/skupperproject/skupper-router/pull/1100>

scripts/zigcc/go.mod

@@ -0,0 +1,3 @@
+module zigcc
+
+go 1.24

scripts/zigcc/test/Dockerfile

@@ -0,0 +1,35 @@
+FROM quay.io/centos/centos:stream9
+
+# Don't install the GCC compiler, we will inject our own CC
+#RUN dnf install -y gcc
+
+# Check CC is in fact injected
+RUN /bin/bash <<'EOF'
+set -Eeuxo pipefail
+ls ${CC}
+${CC} --version
+EOF
+
+#RUN /mnt/zig clang -x c /dev/null --no-default-config -target s390x-unknown-linux5.10.0-gnu2.34.0 -mhard-float -fno-PIE -fPIC -gdwarf-4 -gdwarf32 -fno-lto -MD -MV -MF /root/.cache/zig/tmp/4ee4407ea328f5c8-null.o.d -fhosted -fno-omit-frame-pointer
+#RUN /mnt/zig cc -target s390x-linux-gnu.2.34 --sysroot / -isystem /usr/include -L/usr/lib64 -isystem /usr/local/include -L/usr/local/lib64 -dM -E -x c /dev/null
+RUN /mnt/zig -cc1 -triple x86_64-unknown-linux5.10.0-gnu2.34.0 -E -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name abi-note.S -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -fmath-errno -ffp-contract=on -fno-rounding-math
+
+# Create a directory for the program
+RUN mkdir /app
+
+# Copy the C source code into the container
+RUN sh -c 'cat > /app/hello.c' <<EOF
+#include <stdio.h>
+int main() {
+    printf("Hello, World!\\n");
+    return 0;
+}
+EOF
+
+# Compile the C program
+RUN ${CC} /app/hello.c -o /app/hello
+# Check it works
+RUN /app/hello
+
+# Set the entry point to run the compiled program
+CMD ["/app/hello"]

scripts/zigcc/test/run.py

@@ -0,0 +1,46 @@
+#! /usr/bin/env python3
+
+import argparse
+import logging
+import pathlib
+import subprocess
+import sys
+from typing import cast
+
+ROOT_DIR = pathlib.Path(__file__).parent.parent.parent.parent
+
+logging.basicConfig(level=logging.INFO)
+logging.root.name = pathlib.Path(__file__).name
+
+
+class Args(argparse.Namespace):
+    platform: str
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--platform", default="linux/amd64", help="Target platform for the build")
+    args = cast(Args, p.parse_args())
+
+    dockerfile_path = ROOT_DIR / "scripts" / "zigcc" / "test" / "Dockerfile"
+    sandbox_script_path = ROOT_DIR / "scripts" / "sandbox.py"
+
+    return subprocess.call(
+        [sys.executable, str(sandbox_script_path),
+         "--dockerfile", str(dockerfile_path),
+         "--platform", args.platform,
+         "--",
+         "podman", "build",
+         "--no-cache",
+         "--platform", args.platform,
+         "-t", "hello-world-test",
+         # dockerfile path in podman command is required, Dockerfile is not copied to sandbox
+         "-f", str(dockerfile_path),
+         "{};"],
+        # sandbox.py assumes running from repo root
+        cwd=ROOT_DIR
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main())