diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0d0d4f0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!dist diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 420d116..fdc970f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -23,7 +31,6 @@ jobs: pip install -r requirements.txt - name: Build Wheels run: | - mkdir dist python make_wheels.py - name: Show built files run: | @@ -31,7 +38,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: nodejs-pip-wheels - path: dist/ + path: dist if-no-files-found: error retention-days: 1 @@ -42,8 +49,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - nodejs-version: ['16.15.1', '14.19.3', '18.4.0'] + os: [windows-latest, macos-latest] + nodejs-version: ['14.19.3', '16.15.1', '18.4.0'] python-version: ['3.7', '3.8', '3.9', '3.10'] steps: @@ -77,7 +84,79 @@ jobs: pip install dist\nodejs_bin-${{matrix.nodejs-version}}a3-py3-none-win_amd64.whl - name: Test Package run: - python -m nodejs --version - python -m nodejs.npm --version + python -W error -m nodejs --version + python -W error -m nodejs.npm --version + + test-docker: + name: "Test Docker Architecture:${{ matrix.cpu-arch }} OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}" + runs-on: ubuntu-latest + needs: [build-wheels] + strategy: + fail-fast: false + matrix: + os-variant: [alpine, slim-buster, slim-bullseye] + cpu-arch: [linux/amd64, linux/arm64] + python-version: ['3.7', '3.8', '3.9', '3.10'] + nodejs-version: ['14.19.3', '16.15.1', '18.4.0'] + + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + - uses: actions/download-artifact@v3 + with: + name: nodejs-pip-wheels + path: dist + - name: List available wheels + run: | + ls -lah dist + - name: Docker build + run: | + wheel_prefix_except_platform=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none + + case ${{ matrix.cpu-arch }} in + linux/amd64) + python_cpu_arch=x86_64 + python_libc_variant=2_12 + manylinux_variant_year=2010 + ;; + linux/arm64) + python_cpu_arch=aarch64 + python_libc_variant=2_17 + manylinux_variant_year=2014 + ;; + *) + echo "Could not parse the CPU architecture" + exit 1 + ;; + esac + case ${{ matrix.os-variant }} in + alpine) + python_platform=musllinux_1_1_${python_cpu_arch} + ;; + slim-buster | slim-bullseye) + python_platform=manylinux_${python_libc_variant}_${python_cpu_arch}.manylinux${manylinux_variant_year}_${python_cpu_arch} + ;; + *) + echo "Could not parse the OS variant" + exit 1 + ;; + esac + + WHEEL_TO_INSTALL=${wheel_prefix_except_platform}-${python_platform}.whl + echo "WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL}" + docker build \ + -f Dockerfile \ + --platform=${{ matrix.cpu-arch }} \ + --build-arg PYTHON_VERSION=${{ matrix.python-version }} \ + --build-arg OS_VARIANT=${{ matrix.os-variant }} \ + --build-arg WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL} \ + . diff --git a/.gitignore b/.gitignore index d23485d..a46a752 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ nodejs-cmd/*.egg-info .DS_Store env*/ __pycache__/ -*.py[cod] \ No newline at end of file +*.py[cod] +venv +package.json +node_modules +package-lock.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..97f7286 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +ARG PYTHON_VERSION=3.10 +ARG OS_VARIANT=bullseye-slim + +FROM python:${PYTHON_VERSION}-${OS_VARIANT} + +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION} +ARG OS_VARIANT +ENV OS_VARIANT=${OS_VARIANT} + +# This is required should be supplied as a build-arg +ARG WHEEL_TO_INSTALL +RUN test -n "${WHEEL_TO_INSTALL}" || (echo "Must supply WHEEL_TO_INSTALL as build arg"; exit 1) + +COPY dist/${WHEEL_TO_INSTALL} dist/${WHEEL_TO_INSTALL} + +# NodeJS needs libstdc++ to be present +# https://github.com/nodejs/unofficial-builds/#builds +RUN if echo "${OS_VARIANT}" | grep -e "alpine"; then \ + apk add libstdc++; \ + fi + +RUN pip install dist/${WHEEL_TO_INSTALL} + +RUN python -m nodejs --version +RUN python -m nodejs.npm --version diff --git a/make_wheels.py b/make_wheels.py index ed21e0d..6db33ee 100644 --- a/make_wheels.py +++ b/make_wheels.py @@ -1,7 +1,12 @@ import os import hashlib +import pathlib +import io +import subprocess import urllib.request import libarchive +import tempfile +import tarfile from email.message import EmailMessage from wheel.wheelfile import WheelFile from zipfile import ZipInfo, ZIP_DEFLATED @@ -45,8 +50,59 @@ 'linux-x64': 'manylinux_2_12_x86_64.manylinux2010_x86_64', 'linux-armv7l': 'manylinux_2_17_armv7l.manylinux2014_armv7l', 'linux-arm64': 'manylinux_2_17_aarch64.manylinux2014_aarch64', + 'linux-x64-musl': 'musllinux_1_1_x86_64', + 'linux-arm64-musl': 'musllinux_1_1_aarch64' } +# https://github.com/nodejs/unofficial-builds/ +# Versions added here should match the keys above +UNOFFICIAL_NODEJS_BUILDS = {'linux-x64-musl'} +DOCKER_BASED_BUILDS = {"linux-arm64-musl"} + +_mismatched_versions = (UNOFFICIAL_NODEJS_BUILDS|DOCKER_BASED_BUILDS) - set(PLATFORMS.keys()) +if _mismatched_versions: + raise Exception(f"A version mismatch occurred. Check the usage of {_mismatched_versions}") + +def _build_virtual_release_archive(docker_image: str, platform: str) -> bytes: + # Since npm etc are symlinks we dont copy them here -- the python files shim + # to the lib/mode_modules directory where the real implementation lives as nodejs + # shebanged executables + raw_binaries_to_copy = [x.split("bin/")[1] for x in NODE_BINS if x.startswith("bin")] + tarfile_bytes = io.BytesIO() + with tempfile.TemporaryDirectory() as tmpdirname, tarfile.open(fileobj=tarfile_bytes, mode="w") as tar: + subprocess.check_call( + [ + "docker", + "run", + "--rm", + f"--platform={platform}", + f"--volume={tmpdirname}:/external", + "--entrypoint=sh", + docker_image, + "-c", + f""" + mkdir /external/bin + mkdir /external/lib + for raw_binary in {" ".join(raw_binaries_to_copy)}; do + cp -P $(which $raw_binary) /external/bin + done + if [ -d /usr/local/lib/node_modules ]; then + cp -R /usr/local/lib/node_modules /external/lib + fi + """ + ], + ) + + tmpdir_contents = list(pathlib.Path(tmpdirname).glob("**/*")) + for binary in tmpdir_contents: + relative_path = binary.relative_to(tmpdirname) + if binary.is_file(): + tar_info = tar.gettarinfo(name=binary, arcname=str("node" / relative_path)) + with open(binary, "rb") as f: + tar.addfile(tar_info, f) + + tarfile_bytes.seek(0) + return tarfile_bytes.read() class ReproducibleWheelFile(WheelFile): def writestr(self, zinfo, *args, **kwargs): @@ -113,6 +169,11 @@ def write_nodejs_wheel(out_dir, *, node_version, version, platform, archive): entry_points = {} init_imports = [] + # Create the output directory if it does not exist + out_dir_path = pathlib.Path(out_dir) + if not out_dir_path.exists(): + out_dir_path.mkdir(parents=True) + with libarchive.memory_reader(archive) as archive: for entry in archive: entry_name = '/'.join(entry.name.split('/')[1:]) @@ -246,13 +307,16 @@ def main() -> None: """).encode('ascii') contents['nodejs/__init__.py'] = (cleandoc(""" + import sys from .node import path as path, main as main, call as call, run as run, Popen as Popen - {init_imports} + if not '-m' in sys.argv: + {init_imports} __version__ = "{version}" node_version = "{node_version}" """)).format( - init_imports='\n'.join(init_imports), + # Note: two space indentation above and below is necessary to align + init_imports='\n '.join(init_imports), version=version, node_version=node_version, ).encode('ascii') @@ -294,19 +358,27 @@ def make_nodejs_version(node_version, suffix=''): print('Suffix:', suffix) for node_platform, python_platform in PLATFORMS.items(): - print(f'- Making Wheel for {node_platform}') - node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.' + \ - ('zip' if node_platform.startswith('win-') else 'tar.xz') - - try: - with urllib.request.urlopen(node_url) as request: - node_archive = request.read() - print(f' {node_url}') - print(f' {hashlib.sha256(node_archive).hexdigest()}') - except urllib.error.HTTPError as e: - print(f' {e.code} {e.reason}') - print(f' Skipping {node_platform}') - continue + if node_platform in DOCKER_BASED_BUILDS: + docker_image = f"node:{node_version}-alpine" + print(f'- Making Wheel for {node_platform} from docker image {docker_image}') + node_archive = _build_virtual_release_archive(docker_image=docker_image, platform="linux/arm64") + else: + filetype = 'zip' if node_platform.startswith('win-') else 'tar.xz' + if node_platform in UNOFFICIAL_NODEJS_BUILDS: + node_url = f'https://unofficial-builds.nodejs.org/download/release/v{node_version}/node-v{node_version}-{node_platform}.{filetype}' + else: + node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.{filetype}' + + print(f'- Making Wheel for {node_platform} from {node_url}') + try: + with urllib.request.urlopen(node_url) as request: + node_archive: bytes = request.read() + print(f' {node_url}') + print(f' {hashlib.sha256(node_archive).hexdigest()}') + except urllib.error.HTTPError as e: + print(f' {e.code} {e.reason}') + print(f' Skipping {node_platform}') + continue wheel_path = write_nodejs_wheel('dist/', node_version=node_version,