Skip to content

Commit 367e40a

Browse files
jacobdrjacobdr
committed
Add support for testing arm64 variants
1 parent 2248ea2 commit 367e40a

File tree

2 files changed

+111
-29
lines changed

2 files changed

+111
-29
lines changed

‎.github/workflows/test.yaml‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ jobs:
1414
runs-on: ubuntu-latest
1515
steps:
1616
- uses: actions/checkout@v3
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v2
19+
with:
20+
platforms: arm64
21+
- name: Set up Docker Buildx
22+
uses: docker/setup-buildx-action@v2
23+
with:
24+
install: true
1725
- uses: actions/setup-python@v4
1826
with:
1927
python-version: "3.10"
@@ -23,15 +31,17 @@ jobs:
2331
pip install -r requirements.txt
2432
- name: Build Wheels
2533
run: |
26-
mkdir dist
2734
python make_wheels.py
2835
- name: Show built files
2936
run: |
3037
ls -l dist/*
38+
- name: Zip Artifacts
39+
run: |
40+
tar -czvf dist.tgz dist
3141
- uses: actions/upload-artifact@v3
3242
with:
3343
name: nodejs-pip-wheels
34-
path: dist/
44+
path: dist.tgz
3545
if-no-files-found: error
3646
retention-days: 1
3747

@@ -55,7 +65,8 @@ jobs:
5565
- uses: actions/download-artifact@v3
5666
with:
5767
name: nodejs-pip-wheels
58-
path: dist
68+
path: dist.tgz
69+
- run: tar -xzf dist.tgz
5970
- name: Show available wheels
6071
run: |
6172
ls dist
@@ -80,13 +91,14 @@ jobs:
8091
python -W error -m nodejs --version
8192
python -W error -m nodejs.npm --version
8293
test-docker:
83-
name: "Test Docker OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}"
94+
name: "Test Docker Architecture:${{ matrix.cpu-arch }} OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}"
8495
runs-on: ubuntu-latest
8596
needs: [build-wheels]
8697
strategy:
8798
fail-fast: false
8899
matrix:
89100
os-variant: [alpine, slim-buster, slim-bullseye]
101+
cpu-arch: [linux/amd64, linux/arm64]
90102
python-version: ['3.7', '3.8', '3.9', '3.10']
91103
nodejs-version: ['14.19.3', '16.15.1', '18.4.0']
92104

@@ -103,17 +115,44 @@ jobs:
103115
- uses: actions/download-artifact@v3
104116
with:
105117
name: nodejs-pip-wheels
106-
path: dist
118+
path: dist.tgz
119+
- run: tar -xzf dist.tgz
107120
- name: Docker build
108121
run: |
109-
if [[ ${{ matrix.os-variant }} =~ "alpine" ]]; then
110-
WHEEL_TO_INSTALL=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none-musllinux_1_1_x86_64.whl
111-
else
112-
WHEEL_TO_INSTALL=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl
113-
fi
122+
wheel_prefix_except_platform=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none
123+
124+
case ${{ matrix.cpu-arch }} in
125+
linux/amd64)
126+
python_cpu_arch=x86_64
127+
;;
128+
linux/arm64)
129+
python_cpu_arch=aarch64
130+
;;
131+
*)
132+
echo "Could not parse the CPU architecture"
133+
exit 1
134+
;;
135+
esac
136+
137+
case ${{ matrix.os-variant }} in
138+
alpine)
139+
python_platform=musllinux_1_1_${python_cpu_arch}
140+
;;
141+
slim-buster | slim-bullseye)
142+
python_platform=manylinux_2_12_${python_cpu_arch}.manylinux2010_${python_cpu_arch}
143+
;;
144+
*)
145+
echo "Could not parse the OS variant"
146+
exit 1
147+
;;
148+
esac
149+
150+
WHEEL_TO_INSTALL=${wheel_prefix_except_platform}-${python_platform}.whl
114151
echo "WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL}"
152+
115153
docker build \
116154
-f Dockerfile \
155+
--platform=${{ matrix.cpu-arch }} \
117156
--build-arg PYTHON_VERSION=${{ matrix.python-version }} \
118157
--build-arg OS_VARIANT=${{ matrix.os-variant }} \
119158
--build-arg WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL} \

‎make_wheels.py‎

Lines changed: 62 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
import os
22
import hashlib
33
import pathlib
4+
import io
5+
import subprocess
46
import urllib.request
57
import libarchive
8+
import tempfile
69
from email.message import EmailMessage
710
from wheel.wheelfile import WheelFile
8-
from zipfile import ZipInfo, ZIP_DEFLATED
11+
from zipfile import ZipInfo, ZIP_DEFLATED, ZipFile
912
from inspect import cleandoc
1013

1114

@@ -46,17 +49,52 @@
4649
'linux-x64': 'manylinux_2_12_x86_64.manylinux2010_x86_64',
4750
'linux-armv7l': 'manylinux_2_17_armv7l.manylinux2014_armv7l',
4851
'linux-arm64': 'manylinux_2_17_aarch64.manylinux2014_aarch64',
49-
'linux-x64-musl': 'musllinux_1_1_x86_64'
52+
'linux-x64-musl': 'musllinux_1_1_x86_64',
53+
'linux-arm64-musl': 'musllinux_1_1_aarch64'
5054
}
5155

5256
# https://github.com/nodejs/unofficial-builds/
5357
# Versions added here should match the keys above
5458
UNOFFICIAL_NODEJS_BUILDS = {'linux-x64-musl'}
59+
DOCKER_BASED_BUILDS = {"linux-arm64-musl"}
5560

56-
_mismatched_versions = UNOFFICIAL_NODEJS_BUILDS - set(PLATFORMS.keys())
61+
_mismatched_versions = (UNOFFICIAL_NODEJS_BUILDS|DOCKER_BASED_BUILDS) - set(PLATFORMS.keys())
5762
if _mismatched_versions:
5863
raise Exception(f"A version mismatch occurred. Check the usage of {_mismatched_versions}")
5964

65+
def _build_virtual_release_archive(docker_image: str, platform: str) -> bytes:
66+
binaries_to_copy = [x.split("bin/")[1] for x in [*NODE_BINS, *NODE_OTHER_BINS] if x.startswith("bin")]
67+
zip_bytes = io.BytesIO()
68+
with tempfile.TemporaryDirectory() as tmpdirname, ZipFile(zip_bytes, "w") as zip_file:
69+
subprocess.check_call([
70+
"docker",
71+
"run",
72+
"--rm",
73+
f"--platform={platform}",
74+
f"--volume={tmpdirname}:/external",
75+
"--entrypoint=sh",
76+
docker_image,
77+
"-c",
78+
f"""
79+
for binary in {" ".join(binaries_to_copy)}; do
80+
cp $(which $binary) /external
81+
done
82+
chmod 755 /external/*
83+
"""
84+
],
85+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
86+
87+
tmpdir_contents = list(pathlib.Path(tmpdirname).glob("*"))
88+
for binary in tmpdir_contents:
89+
with open(binary, "rb") as f:
90+
file_info = ZipInfo(
91+
filename=f"node/bin/{binary.name}",
92+
)
93+
file_info.external_attr = 0o755 << 16
94+
zip_file.writestr(file_info, f.read())
95+
96+
zip_bytes.seek(0)
97+
return zip_bytes.read()
6098

6199
class ReproducibleWheelFile(WheelFile):
62100
def writestr(self, zinfo, *args, **kwargs):
@@ -186,7 +224,7 @@ def main() -> None:
186224
elif entry_name in NODE_OTHER_BINS and NODE_OTHER_BINS[entry_name][1]:
187225
other_bin = NODE_OTHER_BINS[entry_name][0]
188226
init_imports.append(f'from . import {other_bin} as {other_bin}')
189-
script_name = '/'.join(os.path.normpath(os.path.join(os.path.dirname(entry.name), entry.linkpath)).split('/')[1:])
227+
script_name = '/'.join(os.path.normpath(os.path.join(os.path.dirname(entry.name), entry.linkpath or other_bin)).split('/')[1:])
190228
contents[f'nodejs/{NODE_OTHER_BINS[entry_name][0]}.py'] = cleandoc(f"""
191229
import os, sys
192230
from typing import TYPE_CHECKING
@@ -312,22 +350,27 @@ def make_nodejs_version(node_version, suffix=''):
312350
print('Suffix:', suffix)
313351

314352
for node_platform, python_platform in PLATFORMS.items():
315-
filetype = 'zip' if node_platform.startswith('win-') else 'tar.xz'
316-
if node_platform in UNOFFICIAL_NODEJS_BUILDS:
317-
node_url = f'https://unofficial-builds.nodejs.org/download/release/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
353+
if node_platform in DOCKER_BASED_BUILDS:
354+
docker_image = f"node:{node_version}-alpine"
355+
print(f'- Making Wheel for {node_platform} from docker image {docker_image}')
356+
node_archive = _build_virtual_release_archive(docker_image=docker_image, platform="linux/arm64")
318357
else:
319-
node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
320-
321-
print(f'- Making Wheel for {node_platform} from {node_url}')
322-
try:
323-
with urllib.request.urlopen(node_url) as request:
324-
node_archive = request.read()
325-
print(f' {node_url}')
326-
print(f' {hashlib.sha256(node_archive).hexdigest()}')
327-
except urllib.error.HTTPError as e:
328-
print(f' {e.code} {e.reason}')
329-
print(f' Skipping {node_platform}')
330-
continue
358+
filetype = 'zip' if node_platform.startswith('win-') else 'tar.xz'
359+
if node_platform in UNOFFICIAL_NODEJS_BUILDS:
360+
node_url = f'https://unofficial-builds.nodejs.org/download/release/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
361+
else:
362+
node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
363+
364+
print(f'- Making Wheel for {node_platform} from {node_url}')
365+
try:
366+
with urllib.request.urlopen(node_url) as request:
367+
node_archive: bytes = request.read()
368+
print(f' {node_url}')
369+
print(f' {hashlib.sha256(node_archive).hexdigest()}')
370+
except urllib.error.HTTPError as e:
371+
print(f' {e.code} {e.reason}')
372+
print(f' Skipping {node_platform}')
373+
continue
331374

332375
wheel_path = write_nodejs_wheel('dist/',
333376
node_version=node_version,

0 commit comments

Comments
 (0)