Manylinux 2_24 and 2_28 wheels for aarch64 (ARM64) and x86_64 (#56)

* Updated CI cfg, setup.py
* Re-generated C files with latest cython
* Added manylinux 2_24 and 2_28 docker builds aarch64 (ARM64) and x86_64
* Updated wheel openssl version. Updated build scripts
* Updated ci cfgs
* Updated dockerfiles
* Updated setup.py

Author: pkittenis

.circleci/config.yml

@@ -23,7 +23,7 @@ jobs:
     parameters:
       python_ver:
         type: string
-        default: "3.6"
+        default: "3.9"
     docker:
       - image: circleci/python:<< parameters.python_ver >>
     steps:
@@ -42,13 +42,14 @@ jobs:
           key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >>
       - run:
           command: |
-            python setup.py build_ext --inplace
+            pip install -e .
             eval "$(ssh-agent -s)"
           name: Build
       - run:
           command: |
             ls -lhtr ssh/
             pwd
+            ldd -r ssh/session*.so
             pytest tests
             flake8 ssh
             python setup.py sdist
@@ -62,7 +63,6 @@ jobs:
     parameters:
       xcode_ver:
         type: string
-        default: "11.6.0"
     macos:
       xcode: << parameters.xcode_ver >>
     environment:
@@ -89,7 +89,7 @@ jobs:
 
   manylinux-x86_64:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-2004:202201-02
     steps: &manylinux-steps
       - checkout
       - run:
@@ -109,10 +109,10 @@ jobs:
           name: Deps
           command: |
             sudo apt-get install python3-pip
-            pyenv global 3.7.0 || pyenv global 3.9.1
             pip install -U pip
             pip install twine
             which twine
+            which python3
       - python/save-cache:
           key: manylinuxdepsv6-{{ .Branch }}.{{ arch }}
           dependency-file: requirements_dev.txt
@@ -123,28 +123,34 @@ jobs:
               echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin;
             fi
             ./ci/build-manylinux.sh
+      - run:
+          name: sdist
+          command: |
+            sudo chown circleci -R *
+            python3 setup.py sdist
       - run:
           name: Upload Wheels
           command: |
             twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASSWORD dist/* wheelhouse/*
 
-  manylinux2014-aarch64:
+  manylinux-aarch64:
     machine:
       image: ubuntu-2004:202101-01
     resource_class: arm.medium
     steps: *manylinux-steps
 
 workflows:
-  version: 2.1
+  version: 2
   main:
     jobs:
       - python_test:
           matrix:
             parameters:
               python_ver:
                 - "3.6"
-                - "3.7"
                 - "3.8"
+                - "3.9"
+                - "3.10"
           filters:
             tags:
               ignore: /.*/
@@ -155,14 +161,13 @@ workflows:
               only: /.*/
             branches:
               ignore: /.*/
-      - manylinux2014-aarch64: *manylinux-wf
+      - manylinux-aarch64: *manylinux-wf
       - osx:
           matrix:
             parameters:
               xcode_ver:
+                - "14.0.0"
                 - "13.1.0"
-                - "13.0.0"
-                - "12.5.1"
           context: Docker
           filters:
             tags:

_setup_libssh.py

@@ -41,7 +41,10 @@ def build_ssh():
         os.symlink('lib', 'local/lib64')
 
     os.chdir('src')
-    check_call('cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../local -DWITH_GSSAPI=ON ../libssh',
+    check_call("""cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../local \
+    -DWITH_GSSAPI=ON \
+    -DWITH_EXAMPLES=OFF \
+    ../libssh""",
                shell=True, env=os.environ)
     check_call(['make', '-j%s' % (cpu_count(),), 'all', 'install'])
     os.chdir('..')

ci/appveyor/build_ssh.bat

@@ -9,6 +9,7 @@ cmake ..\libssh                                    ^
       -DZLIB_LIBRARY=C:/zlib/lib/zlibstatic.lib    ^
       -DZLIB_INCLUDE_DIR=C:/zlib/include           ^
       -DWITH_GSSAPI=ON                             ^
+      -DWITH_EXAMPLES=OFF                          ^
       -DOPENSSL_ROOT_DIR=%OPENSSL_DIR%
 
 

ci/build-manylinux.sh

@@ -16,23 +16,44 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 
-docker_tag="parallelssh/ssh-manylinux"
-docker_files=("ci/docker/manylinux/Dockerfile" "ci/docker/manylinux/Dockerfile.2014_x86_64")
+docker_repo="parallelssh/ssh-manylinux"
+docker_files=(
+              "ci/docker/manylinux/Dockerfile"
+              "ci/docker/manylinux/Dockerfile.2014_x86_64"
+              "ci/docker/manylinux/Dockerfile.manylinux_2_24_x86_64"
+              "ci/docker/manylinux/Dockerfile.manylinux_2_28_x86_64"
+              )
 
 rm -rf local build ssh/libssh.* ssh/*.so
 python ci/appveyor/fix_version.py .
 
 if [[ $(uname -m) == "aarch64" ]]; then
-    docker_tag=${docker_tag}:aarch64
-    docker_files=("ci/docker/manylinux/Dockerfile.aarch64")
+    docker_files=(
+                  "ci/docker/manylinux/Dockerfile.aarch64"
+                  "ci/docker/manylinux/Dockerfile.aarch64_2_24"
+                  "ci/docker/manylinux/Dockerfile.aarch64_2_28"
+                  )
 fi
 
 for docker_file in "${docker_files[@]}"; do
     if [[ ${docker_file} == "ci/docker/manylinux/Dockerfile_2014_x86_64" ]]; then
-        docker_tag="${docker_tag}:2014_x86_64"
+        docker_tag="${docker_repo}:2014_x86_64"
+    elif [[ ${docker_file} == "ci/docker/manylinux/Dockerfile" ]]; then
+        docker_tag="${docker_repo}:2010_x86_64"
+    elif [[ ${docker_file} == "ci/docker/manylinux/Dockerfile.manylinux_2_24_x86_64" ]]; then
+        docker_tag="${docker_repo}:2_24_x86_64"
+    elif [[ ${docker_file} == "ci/docker/manylinux/Dockerfile.manylinux_2_28_x86_64" ]]; then
+        docker_tag="${docker_repo}:2_28_x86_64"
+    elif [[ ${docker_file} == "ci/docker/manylinux/Dockerfile.aarch64" ]]; then
+        docker_tag="${docker_repo}:2014_aarch64"
+    elif [[ ${docker_file} == "ci/docker/manylinux/Dockerfile.aarch64_2_24" ]]; then
+        docker_tag="${docker_repo}:2_24_aarch64"
+    elif [[ ${docker_file} == "ci/docker/manylinux/Dockerfile.aarch64_2_28" ]]; then
+        docker_tag="${docker_repo}:2_28_aarch64"
     fi
+    echo "Docker tag is ${docker_tag}"
     docker pull $docker_tag || echo
-    docker build --pull --cache-from $docker_tag ci/docker/manylinux -t $docker_tag -f ${docker_file}
+    docker build --pull --cache-from $docker_tag ci/docker/manylinux -t $docker_tag -f "${docker_file}"
     if [[ -z "${CIRCLE_PULL_REQUEST}" ]]; then docker push $docker_tag; fi
     docker run --rm -v "$(pwd)":/io $docker_tag /io/ci/build-wheels.sh
     ls wheelhouse/

ci/build-wheels.sh

@@ -15,9 +15,10 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-
 # Compile wheels
 rm -rf /io/build
+# For testing
+#for PYBIN in $(ls -1d /opt/python/cp36-cp36m/bin | grep -v cpython); do
 for PYBIN in $(ls -1d /opt/python/*/bin | grep -v cpython); do
     "${PYBIN}/pip" wheel /io/ -w wheelhouse/
 done
@@ -28,6 +29,7 @@ for whl in wheelhouse/*.whl; do
 done
 
 # Install packages and test
+#for PYBIN in $(ls -1d /opt/python/cp36-cp36m/bin | grep -v cpython); do
 for PYBIN in $(ls -1d /opt/python/*/bin | grep -v cpython); do
     "${PYBIN}/pip" install ssh-python --no-index -f /io/wheelhouse
     (cd "$HOME"; "${PYBIN}/python" -c 'from ssh.session import Session; Session()')

ci/docker/manylinux/Dockerfile

@@ -1,6 +1,6 @@
 FROM quay.io/pypa/manylinux2010_x86_64
 
-ENV OPENSSL openssl-1.1.1g
+ENV OPENSSL openssl-1.1.1q
 ENV LIBSSH 0.9.6
 ENV KRB 1.18.4
 ENV SYSTEM_LIBSSH 1
@@ -27,7 +27,7 @@ RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
 # Libssh
 RUN mkdir -p build_libssh && cd build_libssh && \
     cmake3 ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
-          -DWITH_GSS_API=ON && \
+          -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
 RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz

ci/docker/manylinux/Dockerfile.2014_x86_64

@@ -1,6 +1,6 @@
 FROM quay.io/pypa/manylinux2014_x86_64
 
-ENV OPENSSL openssl-1.1.1g
+ENV OPENSSL openssl-1.1.1q
 ENV LIBSSH 0.9.6
 ENV KRB 1.18.4
 ENV SYSTEM_LIBSSH 1
@@ -27,7 +27,7 @@ RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
 # Libssh
 RUN mkdir -p build_libssh && cd build_libssh && \
     cmake3 ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
-          -DWITH_GSS_API=ON && \
+          -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
 RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz

ci/docker/manylinux/Dockerfile.aarch64

@@ -1,13 +1,12 @@
 FROM quay.io/pypa/manylinux2014_aarch64
 
-ENV OPENSSL openssl-1.1.1g
+ENV OPENSSL openssl-1.1.1q
 ENV LIBSSH 0.9.6
 ENV KRB 1.18.4
 ENV SYSTEM_LIBSSH 1
 ENV CFLAGS "-g0 -s"
 
-RUN yum install epel-release -y
-RUN yum install zlib-devel cmake3 -y
+RUN yum install epel-release -y && yum install zlib-devel cmake3 -y
 
 ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
 ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
@@ -28,7 +27,7 @@ RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
 # Libssh
 RUN mkdir -p build_libssh && cd build_libssh && \
     cmake3 ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
-          -DWITH_GSS_API=ON && \
+          -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
     make -j6 install/strip
 
 RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz

ci/docker/manylinux/Dockerfile.aarch64_2_24

@@ -0,0 +1,36 @@
+FROM quay.io/pypa/manylinux_2_24_aarch64
+
+ENV OPENSSL openssl-1.1.1q
+ENV LIBSSH 0.9.6
+ENV KRB 1.18.4
+ENV SYSTEM_LIBSSH 1
+ENV CFLAGS "-g0 -s"
+ENV OPENSSL_ROOT_DIR /usr/lib
+
+RUN apt-get update -y && apt-get install zlib1g-dev cmake -y
+
+ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
+ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+
+RUN tar -xzf ${OPENSSL}.tar.gz
+# Openssl
+RUN cd ${OPENSSL} && \
+    ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
+    make -j6 && make install_sw
+
+# Kerberos
+RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+    ./configure && \
+    make -j6 && \
+    make install
+
+# Libssh
+RUN mkdir -p build_libssh && cd build_libssh && \
+    cmake ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+          -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
+    make -j6 install/strip
+
+RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+
+VOLUME /var/cache

ci/docker/manylinux/Dockerfile.aarch64_2_28

@@ -0,0 +1,36 @@
+FROM quay.io/pypa/manylinux_2_28_aarch64
+
+ENV OPENSSL openssl-1.1.1q
+ENV LIBSSH 0.9.6
+ENV KRB 1.18.4
+ENV SYSTEM_LIBSSH 1
+ENV CFLAGS "-g0 -s"
+ENV OPENSSL_ROOT_DIR /usr/lib
+
+RUN yum install zlib-devel cmake3 -y
+
+ADD libssh-${LIBSSH}.tar.xz libssh.tar.xz
+ADD https://www.openssl.org/source/${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
+ADD krb5-${KRB}.tar.xz krb5-${KRB}.tar.xz
+
+RUN tar -xzf ${OPENSSL}.tar.gz
+# Openssl
+RUN cd ${OPENSSL} && \
+    ./config --prefix=/usr --openssldir=/usr/openssl threads shared && \
+    make -j6 && make install_sw
+
+# Kerberos
+RUN cd krb5-${KRB}.tar.xz/krb5-${KRB}/src && \
+    ./configure && \
+    make -j6 && \
+    make install
+
+# Libssh
+RUN mkdir -p build_libssh && cd build_libssh && \
+    cmake ../libssh.tar.xz/libssh-${LIBSSH} -DCMAKE_BUILD_TYPE=Release \
+          -DWITH_GSSAPI=ON -DWITH_EXAMPLES=OFF && \
+    make -j6 install/strip
+
+RUN rm -rf ${OPENSSL}* libssh build_libssh krb5-${KRB}.tar.xz
+
+VOLUME /var/cache