Merge branch 'dev' into feature/aks-fix-packaging-test

Author: anonymous-akorn

.github/workflows/cpp-packaging.yml

@@ -69,8 +69,8 @@ jobs:
           binutils_version: "2.35.1"
         - os: macos-latest
           tools_platform: darwin
-          # On Mac, use an older Binutils to avoid corrupted libraries.
-          binutils_version: "2.28.1"
+          # Binutils 2.35.1 released Sep 19, 2020
+          binutils_version: "2.35.1"
     steps:
       - name: setup Xcode version (macos)
         if: runner.os == 'macOS'

.github/workflows/desktop.yml

@@ -88,6 +88,7 @@ jobs:
           python scripts/gha/install_prereqs_desktop.py
 
       - name: Build SDK
+        shell: bash
         run: |
           python scripts/gha/build_desktop.py --build_tests --arch "${{ matrix.architecture }}" --config "${{ matrix.build_type }}" --msvc_runtime_library "${{ matrix.msvc_runtime }}"
 

.github/workflows/integration_tests.yml

@@ -15,6 +15,10 @@ on:
         description: 'CSV of VMs to run on'
         default: 'ubuntu-latest,windows-latest,macos-latest'
         required: true
+      desktop_ssl_variants:
+        description: 'CSV of desktop SSL variants to use'
+        default: 'openssl,boringssl'
+        required: true
       android_device:
         description: 'Android device model'
         default: 'flame'
@@ -36,6 +40,7 @@ jobs:
     outputs:
       matrix_os: ${{ steps.set-matrix-os.outputs.matrix_os }}
       matrix_platform: ${{ steps.set-matrix-os.outputs.matrix_platform }}
+      matrix_ssl: ${{ steps.set-matrix-os.outputs.matrix_ssl }}
     steps:
     - id: set-matrix-os
       # e.g. 'ubuntu-latest,macos-latest' -> '["ubuntu-latest","macos-latest"]'
@@ -44,20 +49,27 @@ jobs:
         echo "::set-output name=matrix_os::${OS_JSON}"
         PLATFORM_JSON=[\"$(echo ${{ github.event.inputs.platforms }} | sed 's/,/","/g')\"]
         echo "::set-output name=matrix_platform::${PLATFORM_JSON}"
+        SSL_JSON=[\"$(echo ${{ github.event.inputs.desktop_ssl_variants }} | sed 's/,/","/g')\"]
+        echo "::set-output name=matrix_ssl::${SSL_JSON}"
   tests:
-    name: ${{ matrix.os }}-${{ matrix.target_platform }}
+    name: ${{ matrix.os }}-${{ matrix.target_platform }}-${{ matrix.ssl_variant }}
     needs: prepare_matrix
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ${{ fromJson(needs.prepare_matrix.outputs.matrix_os) }}
         target_platform: ${{ fromJson(needs.prepare_matrix.outputs.matrix_platform) }}
+        ssl_variant: ${{ fromJson(needs.prepare_matrix.outputs.matrix_ssl) }}
         exclude:
           - os: ubuntu-latest
             target_platform: iOS
           - os: windows-latest
             target_platform: iOS
+          - target_platform: iOS
+            ssl_variant: boringssl
+          - target_platform: Android
+            ssl_variant: boringssl
 
     steps:
       - uses: actions/checkout@v2
@@ -117,8 +129,14 @@ jobs:
           python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
 
       - name: Build integration tests
+        shell: bash
         run: |
-          python scripts/gha/build_testapps.py --t ${{ github.event.inputs.apis }} --p ${{ matrix.target_platform }} --output_directory ${{ github.workspace }} --use_vcpkg --noadd_timestamp
+          # Default SSL is openssl.
+          ssl_option=
+          if [[ "${{ matrix.ssl_variant }}" == "boringssl" ]]; then
+            ssl_option=--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON
+          fi
+          python scripts/gha/build_testapps.py --t ${{ github.event.inputs.apis }} --p ${{ matrix.target_platform }} --output_directory ${{ github.workspace }} --use_vcpkg --noadd_timestamp ${ssl_option}
 
       - name: Run desktop integration tests
         if: matrix.target_platform == 'Desktop' && !cancelled()

CMakeLists.txt

@@ -66,6 +66,8 @@ option(FIREBASE_CPP_BUILD_PACKAGE
        "Bundle the Firebase C++ libraries into a zip file." OFF)
 option(FIREBASE_CPP_USE_PRIOR_GRADLE_BUILD
         "When building with Gradle, use the previously built libraries." OFF)
+option(FIREBASE_USE_BORINGSSL
+        "Build against BoringSSL instead of using your system's OpenSSL." OFF)
 
 set(FIREBASE_ANDROID_STL "" CACHE STRING "STL implementation to use.")
 if (NOT FIREBASE_ANDROID_STL STREQUAL "")
@@ -115,6 +117,13 @@ else()
   set(DESKTOP OFF)
 endif()
 
+if(DESKTOP AND NOT MSVC AND NOT APPLE)
+  # Linux-specific option.
+  add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+endif()
+
 # Set directories needed by the Firebase subprojects
 # Directory to store generated files.
 set(FIREBASE_GEN_FILE_DIR ${CMAKE_BINARY_DIR}/generated)
@@ -174,6 +183,87 @@ else()
   set(FIRESTORE_SOURCE_DIR ${FIREBASE_BINARY_DIR}/external/src/firestore)
 endif()
 
+if(DESKTOP)
+  # Use the static versions of the OpenSSL libraries.
+  set(OPENSSL_USE_STATIC_LIBS TRUE)
+  if (MSVC)
+    # Get the correct version of the OpenSSL libraries based on building for MT.
+    if ("${CMAKE_CXX_FLAGS_DEBUG}" MATCHES "/MTd" OR
+        "${CMAKE_CXX_FLAGS_RELEASE}" MATCHES "/MT")
+      set(OPENSSL_MSVC_STATIC_RT TRUE)
+    else()
+      set(OPENSSL_MSVC_STATIC_RT FALSE)
+    endif()
+  endif()
+
+  if(FIREBASE_USE_BORINGSSL)
+    # Use BoringSSL instead of OpenSSL.
+    set(BORINGSSL_ROOT_DIR ${PROJECT_BINARY_DIR}/external/src/boringssl/src CACHE STRING "" FORCE)
+    set(BORINGSSL_BINARY_DIR ${PROJECT_BINARY_DIR}/external/src/boringssl-build CACHE STRING "" FORCE)
+    set(OPENSSL_ROOT_DIR ${BORINGSSL_ROOT_DIR} CACHE STRING "" FORCE)
+
+    # The call below to build_external_dependencies will make sure that these
+    # libraries exist before the libraries are imported via add_library.
+    if (MSVC)
+      if (CMAKE_BUILD_TYPE)
+        set(BORINGSSL_LIB_SUBDIR "${CMAKE_BUILD_TYPE}")
+      else()
+        set(BORINGSSL_LIB_SUBDIR "Debug")
+      endif()
+      set(OPENSSL_SSL_LIBRARY ${BORINGSSL_BINARY_DIR}/ssl/${BORINGSSL_LIB_SUBDIR}/ssl.lib CACHE FILEPATH "" FORCE)
+      set(OPENSSL_CRYPTO_LIBRARY ${BORINGSSL_BINARY_DIR}/crypto/${BORINGSSL_LIB_SUBDIR}/crypto.lib CACHE FILEPATH "" FORCE)
+    else()
+      set(OPENSSL_SSL_LIBRARY ${BORINGSSL_BINARY_DIR}/ssl/libssl.a CACHE FILEPATH "" FORCE)
+      set(OPENSSL_CRYPTO_LIBRARY ${BORINGSSL_BINARY_DIR}/crypto/libcrypto.a CACHE FILEPATH "" FORCE)
+    endif()
+  endif()
+endif()
+
+
+if(DESKTOP)
+  message(STATUS "Building external project dependencies...")
+  build_external_dependencies()
+  message(STATUS "Build of external project dependencies complete.")
+  
+  if(FIREBASE_USE_BORINGSSL)
+    set(OPENSSL_FOUND TRUE CACHE BOOL "" FORCE)
+    set(OPENSSL_NO_ASM TRUE)  # Force cross-platform BoringSSL, no ASM.
+    set(OPENSSL_INCLUDE_DIR ${BORINGSSL_ROOT_DIR}/include CACHE PATH "" FORCE)
+    set(OPENSSL_CRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY})
+    set(OPENSSL_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY})
+    set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARIES})
+    set(OPENSSL_VERSION '1.1.0' CACHE STRING "" FORCE)
+  
+    add_library(OpenSSL::SSL STATIC IMPORTED)
+    add_library(OpenSSL::Crypto STATIC IMPORTED)
+    set_target_properties(OpenSSL::SSL PROPERTIES
+      IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}"
+      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}"
+      INTERFACE_LINK_LIBRARIES OpenSSL::Crypto
+    )
+  
+    set_target_properties(OpenSSL::Crypto PROPERTIES
+      IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}"
+      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}"
+    )
+    # Now if we do find_package(OpenSSL) it should give us BoringSSL.
+    find_package(OpenSSL)
+  
+    if(NOT "${OPENSSL_INCLUDE_DIR}" MATCHES boringssl OR
+       NOT "${OPENSSL_SSL_LIBRARY}" MATCHES boringssl OR
+       NOT "${OPENSSL_CRYPTO_LIBRARY}" MATCHES boringssl)
+      message(FATAL_ERROR "BoringSSL was not configured correctly.\nINCLUDE_DIR=${OPENSSL_INCLUDE_DIR}\nSSL_LIBRARY=${OPENSSL_SSL_LIBRARY}\nCRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}")
+    endif()
+  else()
+    # Don't use BoringSSL, use OpenSSL. If you are linking against the libraries directly
+    # from source, you probably want this instead.
+    #
+    # If the find_package fails to find OpenSSL, set OPENSSL_ROOT_DIR to OpenSSL'S install
+    # location on your system.
+    find_package(OpenSSL REQUIRED)
+  endif()
+endif()
+
 # Include Firestore's external build early to resolve conflicts on packages.
 if(FIRESTORE_USE_EXTERNAL_CMAKE_BUILD)
   set(FIRESTORE_BINARY_DIR ${FIRESTORE_SOURCE_DIR}-build)
@@ -223,37 +313,28 @@ endif()
 
 # Some of the external libraries are not used for mobile.
 if(DESKTOP)
-  # Use the static versions of the OpenSSL libraries.
-  set(OPENSSL_USE_STATIC_LIBS TRUE)
-  if (MSVC)
-    # Get the correct version of the OpenSSL libraries based on building for MT.
-    if ("${CMAKE_CXX_FLAGS_DEBUG}" MATCHES "/MT" OR
-        "${CMAKE_CXX_FLAGS_RELEASE}" MATCHES "/MT")
-      set(OPENSSL_MSVC_STATIC_RT TRUE)
-    else()
-      set(OPENSSL_MSVC_STATIC_RT FALSE)
-    endif()
-  endif()
-
   # Build curl as a static library
   set(CURL_STATICLIB ON CACHE BOOL "")
   if (WIN32)
-    set(CMAKE_USE_WINSSL ON CACHE BOOL "")
+    # Enable Windows native SSL/TLS in libcurl.
+    set(CMAKE_USE_SCHANNEL ON CACHE BOOL "")
   endif()
+
+  # Current Curl library defaults to requiring some dependencies we don't need, disable them.
+  set(CMAKE_USE_LIBSSH2 OFF)
+  set(HTTP_ONLY ON)
+  set(BUILD_TESTING OFF)
   add_external_library(curl)
 
   add_external_library(libuv)
 
-  find_package(OpenSSL)
-
   add_external_library(zlib)
 
   add_external_library(uWebSockets)
 
   # Binutils on Mac doesn't support thread-local storage (required by
-  # websockets), but because we only use websockets via the scheduler,
-  # we don't need it.
-  
+  # websockets), but because we only use websockets via the scheduler, we don't
+  # need it. Deactivate this by blanking out the __thread keyword.
   set(websockets_additional_defines "-D__thread=")
 
   # uWebSockets does not come with a CMakeLists file, so define the target.
@@ -304,7 +385,8 @@ if(DESKTOP)
     )
     target_link_libraries(libuWS
       PRIVATE
-        ${OPENSSL_LIBRARIES}
+        OpenSSL::SSL
+        OpenSSL::Crypto
         uv_a
         zlibstatic
     )

app/CMakeLists.txt

@@ -325,6 +325,7 @@ target_compile_definitions(firebase_app
   PRIVATE
     -DINTERNAL_EXPERIMENTAL=1
 )
+
 # firebase_app has a dependency on flatbuffers, which needs to be included.
 target_link_libraries(firebase_app
   PRIVATE

app/rest/CMakeLists.txt

@@ -68,13 +68,19 @@ target_compile_definitions(firebase_rest_lib
     "${rest_additional_defines}"
 )
 
+set(FIREBASE_CURL_DEPENDENCY libcurl)
+if (MSVC)
+  # libcurl on Windows depends on internal Windows SSL via its crypt32 library.
+  set(FIREBASE_CURL_DEPENDENCY ${FIREBASE_CURL_DEPENDENCY} crypt32)
+endif()
+
 # firebase_rest_lib has a dependency on flatbuffers, which needs to be included.
 target_link_libraries(firebase_rest_lib
   PUBLIC
     firebase_app
   PRIVATE
     flatbuffers
-    libcurl
+    ${FIREBASE_CURL_DEPENDENCY}
     zlibstatic
     ${CMAKE_DL_LIBS}
 )

build_scripts/desktop/package.sh

@@ -149,26 +149,28 @@ fi
 
 # Library dependencies to merge. Each should be a whitespace-delimited list of path globs.
 readonly deps_firebase_app="
-*/${prefix}firebase_instance_id*${suffix}.${ext}
-*/${prefix}firebase_rest_lib${suffix}.${ext}
+*/${prefix}firebase_instance_id*.${ext}
+*/${prefix}firebase_rest_lib.${ext}
 "
 readonly deps_hidden_firebase_app="
-*/${subdir}${prefix}curl${suffix}.${ext}
-*/${subdir}${prefix}flatbuffers${suffix}.${ext}
+*/curl-build/lib/${subdir}libcurl${suffix}.${ext}
+*/${subdir}${prefix}flatbuffers.${ext}
 */zlib-build/${subdir}${prefix}z.${ext}
 */zlib-build/${subdir}zlibstatic*.${ext}
-*/vcpkg-libs/libcrypto.${ext}
-*/vcpkg-libs/libssl.${ext}
+*/boringssl-build/crypto/${subdir}${prefix}crypto.${ext}
+*/boringssl-build/ssl/${subdir}${prefix}ssl.${ext}
 */firestore-build/*/leveldb-build*/${prefix}*.${ext}
 */firestore-build/*/nanopb-build*/${prefix}*.${ext}
 "
 readonly deps_hidden_firebase_database="
-*/${subdir}${prefix}uv_a${suffix}.${ext}
-*/${subdir}${prefix}libuWS${suffix}.${ext}
+*/${subdir}${prefix}uv_a.${ext}
+*/${subdir}${prefix}libuWS.${ext}
 "
 readonly deps_hidden_firebase_firestore="
 */firestore-build/Firestore/*/${prefix}*.${ext}
-*/firestore-build/*/grpc-build*/${prefix}*.${ext}
+*/firestore-build/*/grpc-build/${subdir}${prefix}*.${ext}
+*/firestore-build/*/grpc-build/third_party/cares/*${subdir}${prefix}*.${ext}
+*/firestore-build/*/grpc-build/third_party/abseil-cpp/*${subdir}${prefix}*.${ext}
 "
 
 # List of C++ namespaces to be renamed, so as to not conflict with the
@@ -281,13 +283,31 @@ for product in ${product_list[*]}; do
 	    deps_hidden+="${found}"
 	done
     done
+    if [[ "${product}" != "app" ]]; then
+      # For any library other than app, also rename some symbols that were already renamed in app
+      # that are used by other libraries (e.g. zlib is used in Firestore).
+      for dep in ${deps_hidden_firebase_app}; do
+        for found in $(find . -path ${dep}); do
+          if [[ ! -z ${deps_hidden} ]]; then deps_hidden+=","; fi
+          deps_hidden+="${found}"
+        done
+      done
+    fi
     echo -n "${libfile_out}"
     if [[ ! -z ${deps_basenames[*]} ]]; then
-	echo -n " <- ${deps_basenames[*]}"
+	echo -n " <- ${deps[*]}"
     fi
     echo
     outfile="${full_output_path}/${libfile_out}"
     rm -f "${outfile}"
+    if [[ ${verbose} -eq 1 ]]; then
+      echo "${python_cmd}" "${merge_libraries_script}" \
+		    ${merge_libraries_params[*]} \
+		    --output="${outfile}" \
+		    --scan_libs="${allfiles}" \
+		    --hide_c_symbols="${deps_hidden}" \
+		    ${libfile_src} ${deps[*]}
+    fi
     "${python_cmd}" "${merge_libraries_script}" \
 		    ${merge_libraries_params[*]} \
 		    --output="${outfile}" \
@@ -296,3 +316,9 @@ for product in ${product_list[*]}; do
 		    ${libfile_src} ${deps[*]}
 done
 cd "${run_path}"
+
+# Copy Firestore core headers into the package's include directory.
+mkdir -p "${output_package_path}/include/firebase/firestore"
+cp -av \
+   "${built_sdk_path}/external/src/firestore/Firestore/core/include/firebase/firestore/"* \
+   "${output_package_path}/include/firebase/firestore"

cmake/external/CMakeLists.txt

@@ -28,6 +28,7 @@ include(firestore)
 
 # Some of the external dependencies are not needed for mobile.
 if (${FIREBASE_EXTERNAL_PLATFORM} STREQUAL "DESKTOP")
+  include(boringssl)
   include(curl)
   include(libuv)
   include(leveldb)