refactor: use pipx to install auditwheel & cmake (#1143)

Author: mayeut

docker/Dockerfile

@@ -148,15 +148,17 @@ COPY --from=build_git /manylinux-rootfs /
 COPY --from=build_swig /manylinux-rootfs /
 COPY --from=build_cpython /manylinux-rootfs /
 COPY --from=all_python /opt/_internal /opt/_internal/
-COPY build_scripts/finalize.sh build_scripts/update-system-packages.sh \
-                               build_scripts/python-tag-abi-tag.py \
-                               build_scripts/requirements3.6.txt \
-                               build_scripts/requirements3.7.txt \
-                               build_scripts/requirements3.8.txt \
-                               build_scripts/requirements3.9.txt \
-                               build_scripts/requirements3.10.txt \
-                               build_scripts/requirements-tools.txt \
-                               /build_scripts/
+COPY build_scripts/finalize.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/python-tag-abi-tag.py \
+     build_scripts/requirements3.6.txt \
+     build_scripts/requirements3.7.txt \
+     build_scripts/requirements3.8.txt \
+     build_scripts/requirements3.9.txt \
+     build_scripts/requirements3.10.txt \
+     build_scripts/requirements-base-tools.txt \
+     /build_scripts/
+COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
 RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem

docker/build_scripts/finalize.sh

@@ -43,16 +43,8 @@ source $TOOLS_PATH/bin/activate
 
 # Install default packages
 pip install -U --require-hashes -r $MY_DIR/requirements3.9.txt
-# Install certifi and auditwheel
-pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt
-
-# Make auditwheel available in PATH
-ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel
-
-# Make CMake available in PATH
-ln -s $TOOLS_PATH/bin/cmake /usr/local/bin/cmake
-ln -s $TOOLS_PATH/bin/cpack /usr/local/bin/cpack
-ln -s $TOOLS_PATH/bin/ctest /usr/local/bin/ctest
+# Install certifi and pipx
+pip install -U --require-hashes -r $MY_DIR/requirements-base-tools.txt
 
 # Make pipx available in PATH,
 # Make sure when root installs apps, they're also in the PATH
@@ -62,6 +54,7 @@ cat <<EOF > /usr/local/bin/pipx
 set -euo pipefail
 
 if [ \$(id -u) -eq 0 ]; then
+	export PIPX_HOME=/opt/_internal/pipx
 	export PIPX_BIN_DIR=/usr/local/bin
 fi
 ${TOOLS_PATH}/bin/pipx "\$@"
@@ -79,6 +72,14 @@ export SSL_CERT_FILE=/opt/_internal/certs.pem
 # Deactivate the tools virtual environment
 deactivate
 
+# install other tools with pipx
+pushd $MY_DIR/requirements-tools
+for TOOL_PATH in $(find . -type f); do
+	TOOL=$(basename ${TOOL_PATH})
+	pipx install --pip-args="--require-hashes -r" ${TOOL}
+done
+popd
+
 # We do not need the precompiled .pyc and .pyo files.
 clean_pyc /opt/_internal
 

docker/build_scripts/requirements-base-tools.txt

@@ -0,0 +1,34 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=docker/build_scripts/requirements-base-tools.txt requirements-base-tools.in
+#
+argcomplete==1.12.3 \
+    --hash=sha256:291f0beca7fd49ce285d2f10e4c1c77e9460cf823eef2de54df0c0fec88b0d81 \
+    --hash=sha256:2c7dbffd8c045ea534921e63b0be6fe65e88599990d8dc408ac8c542b72a5445
+    # via pipx
+certifi==2021.5.30 \
+    --hash=sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee \
+    --hash=sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8
+    # via -r requirements-base-tools.in
+click==8.0.1 \
+    --hash=sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a \
+    --hash=sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6
+    # via userpath
+packaging==21.0 \
+    --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \
+    --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14
+    # via pipx
+pipx==0.16.3 \
+    --hash=sha256:51fa41281383212db3b2a6906713871edc1a7d597ae387873026402e281a0b25 \
+    --hash=sha256:f8af81adaf55716a44f62197528ed3468e33ce53afa21f3b9fe812319bef851c
+    # via -r requirements-base-tools.in
+pyparsing==2.4.7 \
+    --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
+    --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
+    # via packaging
+userpath==1.6.0 \
+    --hash=sha256:43569129eeecec6c567bcbd47120c161a43344712417de013b754191961426db \
+    --hash=sha256:b2b9a5ca1478ecfa63514b48709d650f48bf7be89f62bd236db556b85b6deff6
+    # via pipx

docker/build_scripts/requirements-tools/auditwheel

@@ -0,0 +1,14 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=docker/build_scripts/requirements-tools/auditwheel .nox/tools/tmp/auditwheel.in
+#
+auditwheel==4.0.0 \
+    --hash=sha256:03a079fe273f42336acdb5953ff5ce7578f93ca6a832b16c835fe337a1e2bd4a \
+    --hash=sha256:96927695ddf27b4edb67291e326908d64ffe272b8a42b9504f283e7ae5ebbc14
+    # via -r .nox/tools/tmp/auditwheel.in
+pyelftools==0.27 \
+    --hash=sha256:5609aa6da1123fccfae2e8431a67b4146aa7fad5b3889f808df12b110f230937 \
+    --hash=sha256:cde854e662774c5457d688ca41615f6594187ba7067af101232df889a6b7a66b
+    # via auditwheel

docker/build_scripts/requirements-tools/cmake

@@ -0,0 +1,17 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=docker/build_scripts/requirements-tools/cmake .nox/tools/tmp/cmake.in
+#
+cmake==3.21.0 \
+    --hash=sha256:0a02fb58d782f403587f119535ef0905941308494547053b97cd7cb4df181a07 \
+    --hash=sha256:10247eced45fd7cd82f8f45700a764822c7a768cbb53d54169ea9a9a79252b84 \
+    --hash=sha256:68889247d5f78848bb87d3dec5716169f918c04b11a900084b3259ddeee64dba \
+    --hash=sha256:6e6083d95872124beea93c3abd405f353ad2d870902866f7c92cdb370621c379 \
+    --hash=sha256:73be00791abbf85a482417f10cd7fd9d352b25a5fbc89ab635a571578042278c \
+    --hash=sha256:76e87e27e889f57cf5a2553754b16a6f16edcb8eadaa638506842c4324bdf4eb \
+    --hash=sha256:cbc1768df88daee24ecdf98f263422a6d8862bbd126084287d80017f7e7af34c \
+    --hash=sha256:cd51962b6021760bc5ac2eea5eaa5a8d8bec5820d6e299dee084f133c3440bba \
+    --hash=sha256:e1f96e8ef7776d413669cec93e07437e201a0b84b325974fc3563edec4f2ad20
+    # via -r .nox/tools/tmp/cmake.in

noxfile.py

@@ -1,11 +1,11 @@
+from pathlib import Path
+
 import nox
-import locale
 
 
 @nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"])
 def compile(session):
     session.install("pip-tools")
-
     session.run(
         "pip-compile",
         "--generate-hashes",
@@ -23,8 +23,22 @@ def tools(session):
     session.run(
         "pip-compile",
         "--generate-hashes",
-        "requirements-tools.in",
+        "requirements-base-tools.in",
         "--upgrade",
         "--output-file",
-        f"docker/build_scripts/requirements-tools.txt",
+        "docker/build_scripts/requirements-base-tools.txt",
     )
+    tools = Path("requirements-tools.in").read_text().split("\n")
+    for tool in tools:
+        if tool.strip() == "":
+            continue
+        tmp_file = Path(session.create_tmp()) / f"{tool}.in"
+        tmp_file.write_text(f"{tool}\n")
+        session.run(
+            "pip-compile",
+            "--generate-hashes",
+            str(tmp_file),
+            "--upgrade",
+            "--output-file",
+            f"docker/build_scripts/requirements-tools/{tool}",
+        )

requirements-base-tools.in

@@ -0,0 +1,2 @@
+certifi
+pipx

requirements-tools.in

@@ -1,4 +1,2 @@
 auditwheel
-certifi
 cmake
-pipx

tests/run_tests.sh

@@ -41,6 +41,7 @@ for PYTHON in /opt/python/*/bin/python; do
 done
 
 # minimal tests for tools that should be present
+auditwheel --version
 autoconf --version
 automake --version
 libtoolize --version