From 9252a6e4d28683c2ccf7d6539c19b6369c523d2e Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 6 Feb 2025 13:59:54 -0700 Subject: [PATCH 01/14] alll changes --- .github/docker/Dockerfile.musl | 18 ++++++++++ .github/scripts/utils.mjs | 63 ++++++++++++++++++++++++---------- .github/workflows/build.yml | 33 +++++++++++++++++- .github/workflows/test.yml | 41 +++++++++++++++++++++- etc/docker.sh | 46 +++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 20 deletions(-) create mode 100644 .github/docker/Dockerfile.musl create mode 100644 etc/docker.sh diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl new file mode 100644 index 0000000..11a894d --- /dev/null +++ b/.github/docker/Dockerfile.musl @@ -0,0 +1,18 @@ +ARG PLATFORM=arm64 +ARG NODE_VERSION=16.20.1 + +FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build + +WORKDIR /zstd +COPY . . + +RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip vim cmake git +RUN npm run install:libmongocrypt +RUN npm run prebuild + +ARG RUN_TEST +RUN if [ -n "$RUN_TEST" ]; then npm test ; else echo "skipping tests" ; fi + +FROM scratch + +COPY --from=build /zstd/prebuilds/ / diff --git a/.github/scripts/utils.mjs b/.github/scripts/utils.mjs index 1a54385..45d408d 100644 --- a/.github/scripts/utils.mjs +++ b/.github/scripts/utils.mjs @@ -1,10 +1,10 @@ // @ts-check -import { execSync } from "child_process"; import path from "path"; import url from 'node:url'; import { spawn } from "node:child_process"; import { once } from "node:events"; +import { execSync } from "child_process"; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); @@ -55,22 +55,25 @@ export function buildLibmongocryptDownloadUrl(ref, platform) { } export function getLibmongocryptPrebuildName() { - const platformMatrix = { - ['darwin-arm64']: 'macos', - ['darwin-x64']: 'macos', - ['linux-ppc64']: 'rhel-71-ppc64el', - ['linux-s390x']: 'rhel72-zseries-test', - ['linux-arm64']: 'ubuntu1804-arm64', - ['linux-x64']: 'rhel-70-64-bit', - ['win32-x64']: 'windows-test' - }; - - const detectedPlatform = `${process.platform}-${process.arch}`; - const prebuild = platformMatrix[detectedPlatform]; - - if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`); - - return prebuild; + const prebuildIdentifierFactory = { + 'darwin': () => 'macos', + 'win32': () => 'windows-test', + 'linux': () => { + const key = `${getLibc()}-${process.arch}`; + return { + ['musl-x64']: 'alpine-amd64-earthly', + ['musl-arm64']: 'alpine-arm64-earthly', + ['glibc-ppc64']: 'rhel-71-ppc64el', + ['glibc-s390x']: 'rhel72-zseries-test', + ['glibc-arm64']: 'ubuntu1804-arm64', + ['glibc-x64']: 'rhel-70-64-bit', + }[key] + } + }[process.platform] ?? (() => { + throw new Error(`Unsupported platform`); + }); + + return prebuildIdentifierFactory(); } /** `xtrace` style command runner, uses spawn so that stdio is inherited */ @@ -86,4 +89,28 @@ export async function run(command, args = [], options = {}) { await once(proc, 'exit'); if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`); -} \ No newline at end of file +} + +/** + * @returns the libc (`musl` or `glibc`), if the platform is linux, otherwise null. + */ +function getLibc() { + if (process.platform !== 'linux') return null; + + /** + * executes `ldd --version`. on Alpine linux, `ldd` and `ldd --version` return exit code 1 and print the version + * info to stderr, but on other platforms, `ldd --version` prints to stdout and returns exit code 0. + * + * So, this script works on both by return stderr if the command returns a non-zero exit code, otherwise stdout. + */ + function lddVersion() { + try { + return execSync('ldd --version', { encoding: 'utf-8' }); + } catch (error) { + return error.stderr; + } + } + + console.error({ ldd: lddVersion() }); + return lddVersion().includes('musl') ? 'musl' : 'glibc'; +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c2e37bd..36d67ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: retention-days: 1 compression-level: 0 - container_builds: + container_builds_glibc: outputs: artifact_id: ${{ steps.upload.outputs.artifact-id }} runs-on: ubuntu-latest @@ -69,3 +69,34 @@ jobs: if-no-files-found: 'error' retention-days: 1 compression-level: 0 + + container_tests_musl: + runs-on: ubuntu-latest + strategy: + matrix: + linux_arch: [amd64, arm64] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - name: Get Full Node.js Version + id: get_nodejs_version + shell: bash + run: | + echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker --debug buildx build --progress=plain --no-cache \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.musl \ + . \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c74b60..7b73c26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: shell: bash run: npm run test - container_tests: + container_tests_glibc: runs-on: ubuntu-latest strategy: matrix: @@ -71,3 +71,42 @@ jobs: --output type=local,dest=./prebuilds,platform-split=false \ -f ./.github/docker/Dockerfile.glibc \ . + + + container_tests_musl: + runs-on: ubuntu-latest + strategy: + matrix: + linux_arch: [amd64, arm64] + node: [16.20.1, 18.x, 20.x, 22.x] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + + - name: Get Full Node.js Version + id: get_nodejs_version + shell: bash + run: | + echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker --debug buildx build --progress=plain --no-cache \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ + --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + --build-arg="RUN_TEST=true" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.musl \ + . \ No newline at end of file diff --git a/etc/docker.sh b/etc/docker.sh new file mode 100644 index 0000000..880ee72 --- /dev/null +++ b/etc/docker.sh @@ -0,0 +1,46 @@ +#! /bin/bash + +# script to aid in local testing of linux platforms +# requires a running docker instance + +# s390x, arm64, amd64 for ubuntu +# amd64 or arm64v8 for alpine +LINUX_ARCH=amd64 + +# 16.20.1+, default 16.20.1 +NODE_VERSION=20.0.0 + +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +PROJECT_DIR=$SCRIPT_DIR/.. + +build_and_test_musl() { + docker buildx create --name builder --bootstrap --use + + docker --debug buildx build --load --progress=plain --no-cache \ + --platform linux/$LINUX_ARCH --output=type=docker \ + --build-arg="PLATFORM=$LINUX_ARCH" \ + --build-arg="NODE_VERSION=$NODE_VERSION" \ + --build-arg="RUN_TEST=true" \ + -f ./.github/docker/Dockerfile.musl -t musl-zstd-base \ + . +} + +build_and_test_glibc() { + docker buildx create --name builder --bootstrap --use + + UBUNTU_VERSION=$(node --print 'Number(process.argv[1].split(`.`).at(0)) > 16 ? `noble` : `bionic`' $NODE_VERSION) + NODE_ARCH=$(node -p 'process.argv[1] === `amd64` && `x64` || process.argv[1]' $LINUX_ARCH) + echo $UBUNTU_VERSION + docker buildx build --progress=plain --no-cache \ + --platform linux/$LINUX_ARCH \ + --build-arg="NODE_ARCH=$NODE_ARCH" \ + --build-arg="NODE_VERSION=$NODE_VERSION" \ + --build-arg="UBUNTU_VERSION=$UBUNTU_VERSION" \ + --build-arg="RUN_TEST=true" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.glibc \ + $PROJECT_DIR +} + +build_and_test_musl +# build_and_test_glibc From b8e93830f4db023051f91068876003af8e1edaf0 Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 6 Feb 2025 16:35:31 -0700 Subject: [PATCH 02/14] last changes --- .github/docker/Dockerfile.musl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl index 11a894d..dda49ea 100644 --- a/.github/docker/Dockerfile.musl +++ b/.github/docker/Dockerfile.musl @@ -3,7 +3,7 @@ ARG NODE_VERSION=16.20.1 FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build -WORKDIR /zstd +WORKDIR /mongodb-client-encryption COPY . . RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip vim cmake git @@ -15,4 +15,4 @@ RUN if [ -n "$RUN_TEST" ]; then npm test ; else echo "skipping tests" ; fi FROM scratch -COPY --from=build /zstd/prebuilds/ / +COPY --from=build /mongodb-client-encryption/prebuilds/ / From d21f4a8aeda50514051f0b13f18ec576667814e2 Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 7 Feb 2025 10:39:04 -0700 Subject: [PATCH 03/14] comments --- .github/workflows/build.yml | 13 +++++++++++-- README.md | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36d67ca..07c2461 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: name: Upload prebuild uses: actions/upload-artifact@v4 with: - name: build-linux-${{ matrix.linux_arch }} + name: build-linux-glibc-${{ matrix.linux_arch }} path: prebuilds/ if-no-files-found: 'error' retention-days: 1 @@ -99,4 +99,13 @@ jobs: --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ --output type=local,dest=./prebuilds,platform-split=false \ -f ./.github/docker/Dockerfile.musl \ - . \ No newline at end of file + . + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: build-linux-musl-${{ matrix.linux_arch }} + path: prebuilds/ + if-no-files-found: "error" + retention-days: 1 + compression-level: 0 \ No newline at end of file diff --git a/README.md b/README.md index 66c7521..1b04ee8 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Below are the platforms that are available as prebuilds on each github release. - s390x - arm64 - x64 +- Linux MUSL 1.1.20 + - arm64 + - x64 - MacOS universal binary - x64 - arm64 From fd3a54e48750e3ff7d92b0f66ecb0e9ce62e00c6 Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 7 Feb 2025 12:41:44 -0700 Subject: [PATCH 04/14] remove vim --- .github/docker/Dockerfile.musl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl index dda49ea..6bee364 100644 --- a/.github/docker/Dockerfile.musl +++ b/.github/docker/Dockerfile.musl @@ -6,7 +6,7 @@ FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build WORKDIR /mongodb-client-encryption COPY . . -RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip vim cmake git +RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip cmake git RUN npm run install:libmongocrypt RUN npm run prebuild From 06782c0a1687e2ace29db80453b91b9b4c45da1d Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 7 Feb 2025 13:24:43 -0700 Subject: [PATCH 05/14] pin deps --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b73c26..15f7af7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,8 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 + with: + image: 'tonistiigi/binfmt@sha256:66e11bea77a5ea9d6f0fe79b57cd2b189b5d15b93a2bdb925be22949232e4e55' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -95,6 +97,8 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt@sha256:66e11bea77a5ea9d6f0fe79b57cd2b189b5d15b93a2bdb925be22949232e4e55 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 417a4f7d515c29da69628d63ce4b0525f32cdb8d Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 7 Feb 2025 13:34:55 -0700 Subject: [PATCH 06/14] pin buildx and buildkit --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15f7af7..8be3610 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + version: v0.18.0 + driver-opts: image=moby/buildkit:v0.17.2 - name: Run Buildx run: | @@ -102,6 +105,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + version: v0.18.0 + driver-opts: image=moby/buildkit:v0.17.2 - name: Run Buildx run: | From 876b8b37f7314454db69385d4ef3d05a73634222 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:07:45 -0700 Subject: [PATCH 07/14] ...test? --- .github/docker/Dockerfile.glibc | 16 +++++++--------- .github/docker/Dockerfile.musl | 19 ++++++------------- .github/docker/Test.dockerfile | 9 +++++++++ .github/workflows/test.yml | 25 ++++++++++++++++++++----- etc/docker.sh | 21 +++++++++++++++++---- 5 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 .github/docker/Test.dockerfile diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc index cfd02fc..fc4225d 100644 --- a/.github/docker/Dockerfile.glibc +++ b/.github/docker/Dockerfile.glibc @@ -11,13 +11,11 @@ ENV PATH=$PATH:/nodejs/bin WORKDIR /mongodb-client-encryption COPY . . -RUN apt-get -qq update && apt-get -qq install -y python3 build-essential git && ldd --version +RUN apt-get -qq update && apt-get -qq install -y build-essential curl git && ldd --version -RUN npm run install:libmongocrypt - -ARG RUN_TEST -RUN if [ -n "$RUN_TEST" ]; then npm test ; else echo "skipping tests" ; fi - -FROM scratch - -COPY --from=build /mongodb-client-encryption/prebuilds/ / +RUN bash install-pyenv.sh +RUN pyenv install 3.10 && pyenv global 3.10 +RUN python --version +RUN git --version +RUN c++ --version +RUN g++ --version diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl index 6bee364..3bbec2e 100644 --- a/.github/docker/Dockerfile.musl +++ b/.github/docker/Dockerfile.musl @@ -1,18 +1,11 @@ -ARG PLATFORM=arm64 +ARG ARCH=arm64 ARG NODE_VERSION=16.20.1 -FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build - -WORKDIR /mongodb-client-encryption -COPY . . +FROM ${ARCH}/node:${NODE_VERSION}-alpine AS dependencies RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip cmake git -RUN npm run install:libmongocrypt -RUN npm run prebuild - -ARG RUN_TEST -RUN if [ -n "$RUN_TEST" ]; then npm test ; else echo "skipping tests" ; fi - -FROM scratch -COPY --from=build /mongodb-client-encryption/prebuilds/ / +RUN python3 --version +RUN git --version +RUN c++ --version +RUN g++ --version diff --git a/.github/docker/Test.dockerfile b/.github/docker/Test.dockerfile new file mode 100644 index 0000000..94d2297 --- /dev/null +++ b/.github/docker/Test.dockerfile @@ -0,0 +1,9 @@ +ARG ARCH +ARG NODE_VERSION +FROM ${ARCH}-alpine-base-node-${NODE_VERSION}:latest + +WORKDIR /mongodb-client-encryption +COPY . . + +RUN npm install --ignore-scripts +RUN npm run install:libmongocrypt -- --skip-bindings diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8be3610..ca750c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -109,14 +109,29 @@ jobs: version: v0.18.0 driver-opts: image=moby/buildkit:v0.17.2 + - name: Init builder + run: docker buildx create --name builder --bootstrap --use + - name: Run Buildx run: | - docker buildx create --name builder --bootstrap --use + ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} + NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} + TAG=$ARCH-alpine-lib-base-node-$NODE_VERSION docker --debug buildx build --progress=plain --no-cache \ --platform linux/${{ matrix.linux_arch }} \ - --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ + --build-arg="ARCH=$ARCH" \ --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ - --build-arg="RUN_TEST=true" \ - --output type=local,dest=./prebuilds,platform-split=false \ + --output type=oci,dest=alpine-node-base,platform-split=false \ -f ./.github/docker/Dockerfile.musl \ - . \ No newline at end of file + -t $ARCH-alpine-lib-base-node-$NODE_VERSION + . + + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}-alpine-lib-base-node-${{ steps.get_nodejs_version.outputs.version }} + path: alpine-node-base + if-no-files-found: 'error' + retention-days: 30 + compression-level: 0 \ No newline at end of file diff --git a/etc/docker.sh b/etc/docker.sh index 880ee72..3c38163 100644 --- a/etc/docker.sh +++ b/etc/docker.sh @@ -13,15 +13,28 @@ NODE_VERSION=20.0.0 SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) PROJECT_DIR=$SCRIPT_DIR/.. +# set -o xtrace + build_and_test_musl() { docker buildx create --name builder --bootstrap --use - docker --debug buildx build --load --progress=plain --no-cache \ - --platform linux/$LINUX_ARCH --output=type=docker \ + BASE_TAG=$LINUX_ARCH-alpine-base-node-$NODE_VERSION + docker --debug buildx build --progress=plain \ + --platform linux/$LINUX_ARCH --output=type=oci,dest=alpine-lib-base \ --build-arg="PLATFORM=$LINUX_ARCH" \ --build-arg="NODE_VERSION=$NODE_VERSION" \ --build-arg="RUN_TEST=true" \ - -f ./.github/docker/Dockerfile.musl -t musl-zstd-base \ + -f ./.github/docker/Dockerfile.musl -t $BASE_TAG \ + . + + cat alpine-lib-base | docker load + + LIBMONGOCRYPT_TAG=$LINUX_ARCH-alpine-libmongocrypt-node-$NODE_VERSION + docker --debug buildx build --progress=plain --pull=false --no-cache \ + --platform linux/$LINUX_ARCH --load \ + --build-arg="ARCH=$LINUX_ARCH" \ + --build-arg="NODE_VERSION=$NODE_VERSION" \ + -f ./.github/docker/Test.dockerfile -t $LIBMONGOCRYPT_TAG \ . } @@ -42,5 +55,5 @@ build_and_test_glibc() { $PROJECT_DIR } -build_and_test_musl +# build_and_test_musl # build_and_test_glibc From 4284c08afff43fbda7ebdfa1d18d01ac6dc775e7 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:08:09 -0700 Subject: [PATCH 08/14] winnow matrix to save CI --- .github/workflows/test.yml | 136 ++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca750c0..c382789 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,74 +8,74 @@ on: name: Test jobs: - host_tests: - strategy: - matrix: - os: [macos-latest, windows-2019] - node: [16.x, 18.x, 20.x, 22.x] - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - cache: "npm" - registry-url: "https://registry.npmjs.org" - - - name: Build with Node.js ${{ matrix.node }} on ${{ matrix.os }} - run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }} - shell: bash - - - name: Test ${{ matrix.os }} - shell: bash - run: npm run test - - container_tests_glibc: - runs-on: ubuntu-latest - strategy: - matrix: - linux_arch: [s390x, arm64, amd64] - node: [16.x, 18.x, 20.x, 22.x] - fail-fast: false - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - - name: Get Full Node.js Version - id: get_nodejs_version - shell: bash - run: | - echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" - echo "ubuntu_version=$(node --print '(+process.version.slice(1).split(`.`).at(0)) > 16 ? `noble` : `bionic`')" >> "$GITHUB_OUTPUT" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: 'tonistiigi/binfmt@sha256:66e11bea77a5ea9d6f0fe79b57cd2b189b5d15b93a2bdb925be22949232e4e55' - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: v0.18.0 - driver-opts: image=moby/buildkit:v0.17.2 - - - name: Run Buildx - run: | - docker buildx create --name builder --bootstrap --use - docker buildx build \ - --platform linux/${{ matrix.linux_arch }} \ - --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ - --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ - --build-arg="UBUNTU_VERSION=${{ steps.get_nodejs_version.outputs.ubuntu_version }}" \ - --build-arg="RUN_TEST=true" \ - --output type=local,dest=./prebuilds,platform-split=false \ - -f ./.github/docker/Dockerfile.glibc \ - . + # host_tests: + # strategy: + # matrix: + # os: [macos-latest, windows-2019] + # node: [16.x, 18.x, 20.x, 22.x] + # fail-fast: false + # runs-on: ${{ matrix.os }} + # steps: + # - uses: actions/checkout@v4 + + # - uses: actions/setup-node@v4 + # with: + # node-version: ${{ matrix.node }} + # cache: "npm" + # registry-url: "https://registry.npmjs.org" + + # - name: Build with Node.js ${{ matrix.node }} on ${{ matrix.os }} + # run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }} + # shell: bash + + # - name: Test ${{ matrix.os }} + # shell: bash + # run: npm run test + + # container_tests_glibc: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # linux_arch: [s390x, arm64, amd64] + # node: [16.x, 18.x, 20.x, 22.x] + # fail-fast: false + # steps: + # - uses: actions/checkout@v4 + + # - uses: actions/setup-node@v4 + # with: + # node-version: ${{ matrix.node }} + + # - name: Get Full Node.js Version + # id: get_nodejs_version + # shell: bash + # run: | + # echo "version=$(node --print 'process.version.slice(1)')" >> "$GITHUB_OUTPUT" + # echo "ubuntu_version=$(node --print '(+process.version.slice(1).split(`.`).at(0)) > 16 ? `noble` : `bionic`')" >> "$GITHUB_OUTPUT" + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + # with: + # image: 'tonistiigi/binfmt@sha256:66e11bea77a5ea9d6f0fe79b57cd2b189b5d15b93a2bdb925be22949232e4e55' + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # with: + # version: v0.18.0 + # driver-opts: image=moby/buildkit:v0.17.2 + + # - name: Run Buildx + # run: | + # docker buildx create --name builder --bootstrap --use + # docker buildx build \ + # --platform linux/${{ matrix.linux_arch }} \ + # --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ + # --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + # --build-arg="UBUNTU_VERSION=${{ steps.get_nodejs_version.outputs.ubuntu_version }}" \ + # --build-arg="RUN_TEST=true" \ + # --output type=local,dest=./prebuilds,platform-split=false \ + # -f ./.github/docker/Dockerfile.glibc \ + # . container_tests_musl: From bef07f2717ab6e1d9eb230d14f008a1e420475a2 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:09:10 -0700 Subject: [PATCH 09/14] fix formatting --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c382789..c008422 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -123,7 +123,7 @@ jobs: --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ --output type=oci,dest=alpine-node-base,platform-split=false \ -f ./.github/docker/Dockerfile.musl \ - -t $ARCH-alpine-lib-base-node-$NODE_VERSION + -t $ARCH-alpine-lib-base-node-$NODE_VERSION \ . - id: upload From ae190bf0f9740684f0c24908f00ae8ac55b47ab1 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:31:33 -0700 Subject: [PATCH 10/14] test...? --- .github/docker/Test.dockerfile | 9 ----- .github/workflows/test.yml | 71 +++++++++++++++++++++++++++++++--- etc/docker.sh | 46 ++++++++++++++-------- test/unit/release.test.ts | 13 +++---- 4 files changed, 101 insertions(+), 38 deletions(-) delete mode 100644 .github/docker/Test.dockerfile diff --git a/.github/docker/Test.dockerfile b/.github/docker/Test.dockerfile deleted file mode 100644 index 94d2297..0000000 --- a/.github/docker/Test.dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG ARCH -ARG NODE_VERSION -FROM ${ARCH}-alpine-base-node-${NODE_VERSION}:latest - -WORKDIR /mongodb-client-encryption -COPY . . - -RUN npm install --ignore-scripts -RUN npm run install:libmongocrypt -- --skip-bindings diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c008422..957883e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -112,7 +112,7 @@ jobs: - name: Init builder run: docker buildx create --name builder --bootstrap --use - - name: Run Buildx + - name: Build base image run: | ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} @@ -120,18 +120,77 @@ jobs: docker --debug buildx build --progress=plain --no-cache \ --platform linux/${{ matrix.linux_arch }} \ --build-arg="ARCH=$ARCH" \ - --build-arg="NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }}" \ + --build-arg="NODE_VERSION=$NODE_VERSION" \ --output type=oci,dest=alpine-node-base,platform-split=false \ -f ./.github/docker/Dockerfile.musl \ -t $ARCH-alpine-lib-base-node-$NODE_VERSION \ . - - id: upload - name: Upload prebuild + - id: upload-base-image + name: Upload base image uses: actions/upload-artifact@v4 with: - name: ${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}-alpine-lib-base-node-${{ steps.get_nodejs_version.outputs.version }} + name: ${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}-alpine-base-node-${{ steps.get_nodejs_version.outputs.version }} path: alpine-node-base if-no-files-found: 'error' retention-days: 30 - compression-level: 0 \ No newline at end of file + compression-level: 0 + + - name: Build libmongocrypt + run: | + docker load --input alpine-node-base + ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} + NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} + TAG=$ARCH-alpine-libmongocrypt-base-node-$NODE_VERSION + docker --debug buildx build --progress=plain --no-cache \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="ARCH=$ARCH" \ + --build-arg="NODE_VERSION=$NODE_VERSION" \ + --output type=oci,dest=alpine-libmongocrypt-base,platform-split=false \ + -f ./.github/docker/AddLibmongocrypt.musl \ + -t $ARCH-alpine-libmongocrypt-node-$NODE_VERSION \ + . + + - id: upload-image-with-libmongocrypt + name: Upload libmongocrypt + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}-alpine-libmongocrypt-node-${{ steps.get_nodejs_version.outputs.version }} + path: alpine-libmongocrypt-base + if-no-files-found: 'error' + retention-days: 30 + compression-level: 0 + + - name: Build Prebuilds + run: | + docker load --input alpine-libmongocrypt-base + ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} + NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} + TAG=$ARCH-alpine-prebuilds-base-node-$NODE_VERSION + docker --debug buildx build --progress=plain --no-cache \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="ARCH=$ARCH" \ + --build-arg="NODE_VERSION=$NODE_VERSION" \ + --output type=oci,dest=alpine-prebuilds-base,platform-split=false \ + -f ./.github/docker/Prebuilds.musl \ + -t $ARCH-alpine-prebuilds-node-$NODE_VERSION \ + . + + - id: upload-images-with-prebuilds + name: Upload libmongocrypt + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}-alpine-prebuilds-node-${{ steps.get_nodejs_version.outputs.version }} + path: alpine-prebuilds-base + if-no-files-found: 'error' + retention-days: 30 + compression-level: 0 + + # - name: Test + # run: | + # docker load --input alpine-prebuilds-base + # ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} + # NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} + # TAG=$ARCH-alpine-prebuilds-base-node-$NODE_VERSION + # docker run --platform linux/${{ matrix.linux_arch }} $ARCH-alpine-prebuilds-node-$NODE_VERSION bash -C "npm test" + \ No newline at end of file diff --git a/etc/docker.sh b/etc/docker.sh index 3c38163..495eaf6 100644 --- a/etc/docker.sh +++ b/etc/docker.sh @@ -19,23 +19,37 @@ build_and_test_musl() { docker buildx create --name builder --bootstrap --use BASE_TAG=$LINUX_ARCH-alpine-base-node-$NODE_VERSION - docker --debug buildx build --progress=plain \ - --platform linux/$LINUX_ARCH --output=type=oci,dest=alpine-lib-base \ - --build-arg="PLATFORM=$LINUX_ARCH" \ - --build-arg="NODE_VERSION=$NODE_VERSION" \ - --build-arg="RUN_TEST=true" \ - -f ./.github/docker/Dockerfile.musl -t $BASE_TAG \ - . + # docker --debug buildx build --progress=plain \ + # --platform linux/$LINUX_ARCH --output=type=oci,dest=alpine-lib-base \ + # --build-arg="ARCH=$LINUX_ARCH" \ + # --build-arg="NODE_VERSION=$NODE_VERSION" \ + # --build-arg="RUN_TEST=true" \ + # -f ./.github/docker/Dockerfile.musl -t $BASE_TAG \ + # . - cat alpine-lib-base | docker load + # cat alpine-lib-base | docker load - LIBMONGOCRYPT_TAG=$LINUX_ARCH-alpine-libmongocrypt-node-$NODE_VERSION - docker --debug buildx build --progress=plain --pull=false --no-cache \ - --platform linux/$LINUX_ARCH --load \ - --build-arg="ARCH=$LINUX_ARCH" \ - --build-arg="NODE_VERSION=$NODE_VERSION" \ - -f ./.github/docker/Test.dockerfile -t $LIBMONGOCRYPT_TAG \ - . + # LIBMONGOCRYPT_TAG=$LINUX_ARCH-alpine-libmongocrypt-node-$NODE_VERSION + # docker --debug buildx build --progress=plain --pull=false \ + # --output=type=oci,dest=alpine-libmongocrypt \ + # --platform linux/$LINUX_ARCH \ + # --build-arg="ARCH=$LINUX_ARCH" \ + # --build-arg="NODE_VERSION=$NODE_VERSION" \ + # -f ./.github/docker/AddLibmongocrypt.dockerfile -t $LIBMONGOCRYPT_TAG \ + # . + + # cat alpine-libmongocrypt | docker load + + LIBMONGOCRYPT_TAG=$LINUX_ARCH-alpine-prebuilds-node-$NODE_VERSION + # docker --debug buildx build --progress=plain --pull=false \ + # --load \ + # --platform linux/$LINUX_ARCH \ + # --build-arg="ARCH=$LINUX_ARCH" \ + # --build-arg="NODE_VERSION=$NODE_VERSION" \ + # -f ./.github/docker/Prebuilds.dockerfile -t $LIBMONGOCRYPT_TAG \ + # . + + docker --debug run $LIBMONGOCRYPT_TAG bash -c "npm test" } build_and_test_glibc() { @@ -55,5 +69,5 @@ build_and_test_glibc() { $PROJECT_DIR } -# build_and_test_musl +build_and_test_musl # build_and_test_glibc diff --git a/test/unit/release.test.ts b/test/unit/release.test.ts index f82c473..f436acd 100644 --- a/test/unit/release.test.ts +++ b/test/unit/release.test.ts @@ -37,21 +37,20 @@ describe(`Release ${packFile}`, function () { } }); - let tarFileList; - beforeEach(() => { + const tarFileSet: Set = new Set(); + before(() => { expect(fs.existsSync(packFile)).to.equal(false); cp.execSync('npm pack', { stdio: 'ignore' }); - tarFileList = []; tar.list({ file: packFile, sync: true, onentry(entry) { - tarFileList.push(entry.path); + tarFileSet.add(entry.path); } }); }); - afterEach(() => { + after(() => { if (process.arch === 'x64') { fs.unlinkSync(packFile); } @@ -59,12 +58,12 @@ describe(`Release ${packFile}`, function () { for (const requiredFile of REQUIRED_FILES) { it(`should contain ${requiredFile}`, () => { - expect(tarFileList).to.includes(requiredFile); + expect(tarFileSet).to.include(requiredFile); }); } it('should not have extraneous files', () => { - const unexpectedFileList = tarFileList.filter(f => !REQUIRED_FILES.some(r => r === f)); + const unexpectedFileList = Array.from(tarFileSet).filter(f => !REQUIRED_FILES.some(r => r === f)); expect(unexpectedFileList).to.have.lengthOf(0, `Extra files: ${unexpectedFileList.join(', ')}`); }); }); From 6fdb3578af7d3f1648bb9d9153e76d25499bb20d Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:32:13 -0700 Subject: [PATCH 11/14] masdf --- .github/docker/AddLibmongocrypt.dockerfile | 9 +++++++++ .github/docker/Prebuilds.dockerfile | 5 +++++ 2 files changed, 14 insertions(+) create mode 100644 .github/docker/AddLibmongocrypt.dockerfile create mode 100644 .github/docker/Prebuilds.dockerfile diff --git a/.github/docker/AddLibmongocrypt.dockerfile b/.github/docker/AddLibmongocrypt.dockerfile new file mode 100644 index 0000000..94d2297 --- /dev/null +++ b/.github/docker/AddLibmongocrypt.dockerfile @@ -0,0 +1,9 @@ +ARG ARCH +ARG NODE_VERSION +FROM ${ARCH}-alpine-base-node-${NODE_VERSION}:latest + +WORKDIR /mongodb-client-encryption +COPY . . + +RUN npm install --ignore-scripts +RUN npm run install:libmongocrypt -- --skip-bindings diff --git a/.github/docker/Prebuilds.dockerfile b/.github/docker/Prebuilds.dockerfile new file mode 100644 index 0000000..1c80ab3 --- /dev/null +++ b/.github/docker/Prebuilds.dockerfile @@ -0,0 +1,5 @@ +ARG ARCH +ARG NODE_VERSION +FROM ${ARCH}-alpine-libmongocrypt-node-${NODE_VERSION}:latest + +RUN npm run prebuild From 56ae3de802ac8b6b018454e5ac84e2418427df35 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:34:17 -0700 Subject: [PATCH 12/14] asf --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 957883e..1e9da3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,6 +125,7 @@ jobs: -f ./.github/docker/Dockerfile.musl \ -t $ARCH-alpine-lib-base-node-$NODE_VERSION \ . + docker load --input alpine-node-base - id: upload-base-image name: Upload base image @@ -138,7 +139,6 @@ jobs: - name: Build libmongocrypt run: | - docker load --input alpine-node-base ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} TAG=$ARCH-alpine-libmongocrypt-base-node-$NODE_VERSION @@ -150,7 +150,8 @@ jobs: -f ./.github/docker/AddLibmongocrypt.musl \ -t $ARCH-alpine-libmongocrypt-node-$NODE_VERSION \ . - + docker load --input alpine-libmongocrypt-base + - id: upload-image-with-libmongocrypt name: Upload libmongocrypt uses: actions/upload-artifact@v4 @@ -163,7 +164,6 @@ jobs: - name: Build Prebuilds run: | - docker load --input alpine-libmongocrypt-base ARCH=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }} NODE_VERSION=${{ steps.get_nodejs_version.outputs.version }} TAG=$ARCH-alpine-prebuilds-base-node-$NODE_VERSION From c339e0a87dc01be127b4523755a24d094aca0688 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:36:13 -0700 Subject: [PATCH 13/14] asdf --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e9da3a..1373138 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,7 +125,8 @@ jobs: -f ./.github/docker/Dockerfile.musl \ -t $ARCH-alpine-lib-base-node-$NODE_VERSION \ . - docker load --input alpine-node-base + + cat alpine-node-base | docker load - id: upload-base-image name: Upload base image From fe5f6893d6c26ec7cbd275d612584a7e8841a232 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 8 Feb 2025 07:37:34 -0700 Subject: [PATCH 14/14] asdf --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1373138..bdba7d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,8 +125,9 @@ jobs: -f ./.github/docker/Dockerfile.musl \ -t $ARCH-alpine-lib-base-node-$NODE_VERSION \ . - - cat alpine-node-base | docker load + + ls + # cat alpine-node-base | docker load - id: upload-base-image name: Upload base image