Fix the broken CI due to the regression of 4.0.1 and some deprecated image and links (#235)

Fixes #234

1. Downgrade the Pulsar image to 4.0.0
2. Upgrade the macOS runner image from 12 to 14

Since the Boost download link is broken now, this PR also fixes the wheel build process by reusing the pre-built libraries to install pulsar-client-cpp on Linux and macOS. Specially, it links the `libpulsarwithdeps.a` instead of `libpulsar.a` and all 3rd-party libraries on macOS.

Author: BewareMyPower

.github/workflows/ci-build-release-wheels.yaml

@@ -58,7 +58,6 @@ jobs:
         uses: docker/setup-qemu-action@v2
 
       - uses: docker/setup-buildx-action@v2
-      - run: build-support/copy-deps-versionfile.sh
 
       - name: Build Manylinux Docker image
         uses: docker/build-push-action@v3
@@ -95,7 +94,7 @@ jobs:
 
   mac-wheels:
     name: Wheel MacOS Universal2 - Py ${{matrix.py.version}}
-    runs-on: macos-12
+    runs-on: macos-14
     timeout-minutes: 300
 
     strategy:
@@ -112,30 +111,8 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
 
-      - name: Dependencies cache
-        uses: actions/cache@v3
-        id: cache-deps
-        with:
-          path: .pulsar-mac-build/deps/install
-          key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }}
-
-      - name: Build dependencies
-        if: steps.cache-deps.outputs.cache-hit != 'true'
-        run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}}
-
-      - name: Pulsar C++ cache
-        uses: actions/cache@v3
-        id: cache-cpp
-        with:
-          path: .pulsar-mac-build/cpp/install
-          key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }}
-
-      - name: Build Pulsar C++ client
-        if: steps.cache-cpp.outputs.cache-hit != 'true'
-        run: pkg/mac/build-pulsar-cpp.sh
-
       - name: Build and test Mac wheels
-        run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}
+        run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3

.github/workflows/ci-pr-validation.yaml

@@ -127,7 +127,6 @@ jobs:
         uses: docker/setup-qemu-action@v2
 
       - uses: docker/setup-buildx-action@v2
-      - run: build-support/copy-deps-versionfile.sh
 
       - name: Build Manylinux Docker image
         uses: docker/build-push-action@v3
@@ -157,7 +156,7 @@ jobs:
   mac-wheels:
     name: Wheel MacOS Universal2 - Py ${{matrix.py.version}}
     needs: unit-tests
-    runs-on: macos-12
+    runs-on: macos-14
     timeout-minutes: 300
 
     strategy:
@@ -169,33 +168,9 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
 
-      - name: Dependencies cache
-        uses: actions/cache@v3
-        id: cache-deps
-        with:
-          path: .pulsar-mac-build/deps/install
-          key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }}
-
-      - name: Install Python deps
-        run: pip3 install -U pyyaml
-
-      - name: Build dependencies
-        if: steps.cache-deps.outputs.cache-hit != 'true'
-        run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}}
-
-      - name: Pulsar C++ cache
-        uses: actions/cache@v3
-        id: cache-cpp
-        with:
-          path: .pulsar-mac-build/cpp/install
-          key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }}
-
-      - name: Build Pulsar C++ client
-        if: steps.cache-cpp.outputs.cache-hit != 'true'
-        run: pkg/mac/build-pulsar-cpp.sh
-
       - name: Build and test Mac wheels
-        run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}
+        run: |
+          pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}}
 
   windows-wheels:
     name: "Python ${{ matrix.python.version }} Wheel on Windows x64"

CMakeLists.txt

@@ -38,13 +38,9 @@ if (MSVC)
 endif ()
 
 if (LINK_STATIC)
-    if (MSVC)
-        find_library(PULSAR_LIBRARY NAMES pulsarWithDeps.lib)
-    else ()
-        find_library(PULSAR_LIBRARY NAMES libpulsar.a)
-    endif ()
+    find_library(PULSAR_LIBRARY NAMES pulsarwithdeps pulsarWithDeps.lib)
     add_definitions("-DPULSAR_STATIC")
-else()
+else ()
     find_library(PULSAR_LIBRARY NAMES pulsar libpulsar)
 endif()
 message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")
@@ -88,43 +84,11 @@ set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module)
 message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")
 
 if (LINK_STATIC AND NOT MSVC)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-
-    # We need to include all the static libs individually because we cannot easily create a universal2 libpulsar.a
-    # with all the deps included.
-    find_package(OpenSSL REQUIRED)
-    message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES})
-
-    find_package(Protobuf REQUIRED)
-    message("Protobuf_LIBRARIES: " ${Protobuf_LIBRARIES})
-
-    find_package(curl REQUIRED)
-    message("CURL_LIBRARIES: " ${CURL_LIBRARIES})
-
-    find_package(zlib)
-    message("ZLIB_LIBRARIES: " ${ZLIB_LIBRARIES})
-
-    find_library(LIB_ZSTD NAMES libzstd.a)
-    message(STATUS "ZStd: ${LIB_ZSTD}")
-    find_library(LIB_SNAPPY NAMES libsnappy.a)
-    message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}")
-
-    set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS}
-                        ${OPENSSL_LIBRARIES}
-                        ${Protobuf_LIBRARIES}
-                        ${ZLIB_LIBRARIES}
-                        ${LIB_ZSTD}
-                        ${LIB_SNAPPY}
-                        ${CURL_LIBRARIES}
-                        )
-
     if (APPLE)
         set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
-        target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS})
+        target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
     else ()
-        if (NOT MSVC)
-          set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc  -static-libstdc++")
-        endif()
+        set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc  -static-libstdc++")
         target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
     endif ()
 elseif (LINK_STATIC) # MSVC

build-support/copy-deps-versionfile.sh

@@ -38,38 +38,15 @@ download_dependency() {
   VERSION=$(grep $DEP $DEP_FILE | sed 's/://' | awk '{print $2}')
 
   case $DEP in
-    "cmake")
-      URL=https://github.com/Kitware/CMake/releases/download/v${VERSION}/cmake-${VERSION}-linux-${ARCH}.tar.gz
-      ;;
     "pulsar-cpp")
       URL=$(pulsar_cpp_base_url $VERSION)/apache-pulsar-client-cpp-${VERSION}.tar.gz
       ;;
     "pybind11")
       URL=https://github.com/pybind/pybind11/archive/refs/tags/v${VERSION}.tar.gz
       ;;
-    "boost")
-      VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g')
-      URL=https://boostorg.jfrog.io/artifactory/main/release/${VERSION}/source/boost_${VERSION_UNDERSCORE}.tar.gz
-      ;;
-    "protobuf")
-      URL=https://github.com/google/protobuf/releases/download/v${VERSION}/protobuf-cpp-${VERSION}.tar.gz
-      ;;
-    "zlib")
-      URL=https://github.com/madler/zlib/archive/v${VERSION}.tar.gz
-      ;;
-    "zstd")
-      URL=https://github.com/facebook/zstd/releases/download/v${VERSION}/zstd-${VERSION}.tar.gz
-      ;;
-    "snappy")
-      URL=https://github.com/google/snappy/archive/refs/tags/${VERSION}.tar.gz
-      ;;
     "openssl")
       URL=https://github.com/openssl/openssl/archive/OpenSSL_$(echo $VERSION | sed 's/\./_/g').tar.gz
       ;;
-    "curl")
-      VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g')
-      URL=https://github.com/curl/curl/releases/download/curl-${VERSION_UNDERSCORE}/curl-${VERSION}.tar.gz
-      ;;
     *)
       echo "Unknown dependency $DEP for version $VERSION"
       exit 1

build-support/dep-url.sh

@@ -25,7 +25,7 @@ cd $SRC_DIR
 
 ./build-support/pulsar-test-service-stop.sh
 
-CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600)
+CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:4.0.0 sleep 3600)
 echo $CONTAINER_ID > .tests-container-id.txt
 
 docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf

build-support/pulsar-test-service-start.sh

@@ -17,13 +17,7 @@
 # under the License.
 #
 
-cmake: 3.24.2
 pulsar-cpp: 3.5.1
 pybind11: 2.10.1
-boost: 1.80.0
-protobuf: 3.20.0
-zlib: 1.2.13
-zstd: 1.5.2
-snappy: 1.1.9
+# The OpenSSL dependency is only used when building Python from source
 openssl: 1.1.1q
-curl: 8.4.0

dependencies.yaml

@@ -21,19 +21,39 @@
 set -e -x
 
 cd /pulsar-client-python
+source build-support/dep-url.sh
 
-PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
-curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
-tar zxf v${PYBIND11_VERSION}.tar.gz
-rm -rf pybind11
-mv pybind11-${PYBIND11_VERSION} pybind11
+# Build cpp wheels
+PULSAR_CPP_VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}')
 
-rm -f CMakeCache.txt CMakeFiles
+if [ $CPP_BINARY_TYPE == "rpm" ]; then
+    if [ $ARCH == "aarch64" ]; then
+        RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-arm64/aarch64
+    else
+        RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-x86_64/x86_64
+    fi
+    curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-1.$ARCH.rpm
+    curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-devel-$PULSAR_CPP_VERSION-1.$ARCH.rpm
+    curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-debuginfo-$PULSAR_CPP_VERSION-1.$ARCH.rpm
+    rpm -ivh *.rpm
+else # apk
+    if [ $ARCH == "aarch64" ]; then
+        APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-arm64/aarch64
+    else
+        APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-x86_64/x86_64
+    fi
+    curl -O -L $APK_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-r0.apk
+    curl -O -L $APK_ROOT_DIR/apache-pulsar-client-dev-$PULSAR_CPP_VERSION-r0.apk
+    apk add --allow-untrusted *.apk
+fi
 
-cmake . \
-      -DCMAKE_BUILD_TYPE=Release
+download_dependency $PWD/dependencies.yaml pybind11
+rm -rf pybind11
+mv pybind11-* pybind11
 
-make -j4
+cmake -B build -DCMAKE_BUILD_TYPE=Release
+cmake --build build -j8
+mv build/lib_pulsar.so .
 
 ./setup.py bdist_wheel