Add integration tests

.github/workflows/integration_tests.yaml

@@ -0,0 +1,51 @@
+name: Integration tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  build_integration_container_and_run_tests:
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v5
+
+    - name: Create build environment
+      uses: mamba-org/setup-micromamba@v2
+      with:
+        environment-file: ./environment-dev.yml
+        environment-name: build_env
+        cache-environment: true
+
+    - name: Configure using CMake
+      run: |
+        cmake -G Ninja \
+          -Bbuild \
+          -DCMAKE_BUILD_TYPE:STRING=RELEASE \
+          -DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
+          -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
+          -DSPARROW_IPC_BUILD_SHARED=ON
+
+    - name: Build file_to_stream target
+      working-directory: build
+      run: cmake --build . --config Release --target file_to_stream
+
+    - name: Build stream_to_file target
+      working-directory: build
+      run: cmake --build . --config Release --target stream_to_file
+
+    - name: Build Docker image
+      run: docker build -t sparrow/integration-tests -f ci/docker/integration.dockerfile .
+
+    - name: Run Integration tests
+      run: |
+        docker run --rm \
+          -e ARCHERY_INTEGRATION_WITH_EXTERNAL_LIBRARY=/workspace/build/bin/RELEASE/ \
+          -e ARCHERY_INTEGRATION_EXTERNAL_LIBRARY_IPC_PRODUCER=true \
+          -e ARCHERY_INTEGRATION_EXTERNAL_LIBRARY_IPC_CONSUMER=true \
+          -v ${{ github.workspace }}:/workspace \
+          -w /arrow-integration sparrow/integration-tests \
+          "/arrow-integration/ci/scripts/integration_arrow.sh /arrow-integration /build"

CMakeLists.txt

@@ -11,8 +11,6 @@ include(CMakeDependentOption)
 list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
 message(DEBUG "CMake module path: ${CMAKE_MODULE_PATH}")
 
-include(external_dependencies)
-
 set(SPARROW_IPC_COMPILE_DEFINITIONS "" CACHE STRING "List of public compile definitions of the sparrow-ipc target")
 
 set(SPARROW_IPC_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -85,11 +83,16 @@ MESSAGE(STATUS "🔧 Build docs: ${SPARROW_IPC_BUILD_DOCS}")
 OPTION(SPARROW_IPC_BUILD_EXAMPLES "Build sparrow-ipc examples" OFF)
 MESSAGE(STATUS "🔧 Build examples: ${SPARROW_IPC_BUILD_EXAMPLES}")
 
+OPTION(SPARROW_IPC_BUILD_INTEGRATION_TESTS "Build sparrow-ipc integration tests" OFF)
+MESSAGE(STATUS "🔧 Build integration tests: ${SPARROW_IPC_BUILD_INTEGRATION_TESTS}")
+
 # Code coverage
 # =============
 OPTION(SPARROW_IPC_ENABLE_COVERAGE "Enable sparrow-ipc test coverage" OFF)
 MESSAGE(STATUS "🔧 Enable coverage: ${SPARROW_IPC_ENABLE_COVERAGE}")
 
+include(external_dependencies)
+
 if(SPARROW_IPC_ENABLE_COVERAGE)
     include(code_coverage)
 endif()
@@ -284,6 +287,13 @@ if(SPARROW_IPC_BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
+# Integration tests
+# =================
+if(SPARROW_IPC_BUILD_INTEGRATION_TESTS)
+    message(STATUS "🔨 Create integration tests targets")
+    add_subdirectory(integration_tests)
+endif()
+
 # Installation
 # ============
 include(GNUInstallDirs)

ci/docker/integration.dockerfile

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM apache/arrow-dev:amd64-conda-integration
+
+ENV ARROW_USE_CCACHE=OFF \
+    ARROW_CPP_EXE_PATH=/build/cpp/debug \
+    BUILD_DOCS_CPP=OFF \
+    ARROW_INTEGRATION_CPP=ON \
+    ARROW_INTEGRATION_CSHARP=OFF \
+    ARROW_INTEGRATION_GO=OFF \
+    ARROW_INTEGRATION_JAVA=OFF \
+    ARROW_INTEGRATION_JS=OFF \
+    ARCHERY_INTEGRATION_WITH_NANOARROW="0" \
+    ARCHERY_INTEGRATION_WITH_RUST="0"
+
+RUN apt update
+
+RUN apt install build-essential git -y
+
+# Clone the arrow monorepo // TODO: change to the official repo
+RUN git clone --depth 1 --branch archery_supports_external_libraries https://github.com/Alex-PLACET/arrow.git /arrow-integration --recurse-submodules
+
+# Build all the integrations
+RUN conda run --no-capture-output \
+    /arrow-integration/ci/scripts/integration_arrow_build.sh \
+    /arrow-integration \
+    /build

cmake/external_dependencies.cmake

@@ -68,7 +68,7 @@ function(find_package_or_fetch)
 endfunction()
 
 set(SPARROW_BUILD_SHARED ${SPARROW_IPC_BUILD_SHARED})
-if(${SPARROW_IPC_BUILD_TESTS})
+if(${SPARROW_IPC_BUILD_TESTS} OR ${SPARROW_IPC_BUILD_INTEGRATION_TESTS})
     set(CREATE_JSON_READER_TARGET ON)
 endif()
 find_package_or_fetch(
@@ -81,7 +81,7 @@ unset(CREATE_JSON_READER_TARGET)
 if(NOT TARGET sparrow::sparrow)
     add_library(sparrow::sparrow ALIAS sparrow)
 endif()
-if(${SPARROW_IPC_BUILD_TESTS})
+if(${SPARROW_IPC_BUILD_TESTS} OR ${SPARROW_IPC_BUILD_INTEGRATION_TESTS})
     find_package_or_fetch(
         PACKAGE_NAME sparrow-json-reader
     )
@@ -123,7 +123,7 @@ if(NOT TARGET lz4::lz4)
     add_library(lz4::lz4 ALIAS lz4)
 endif()
 
-if(SPARROW_IPC_BUILD_TESTS)
+if(${SPARROW_IPC_BUILD_TESTS} OR ${SPARROW_IPC_BUILD_INTEGRATION_TESTS})
     find_package_or_fetch(
         PACKAGE_NAME doctest
         GIT_REPOSITORY https://github.com/doctest/doctest.git

integration_tests/CMakeLists.txt

@@ -0,0 +1,130 @@
+cmake_minimum_required(VERSION 3.28)
+
+# Create executable for file_to_stream integration test
+add_executable(file_to_stream file_to_stream.cpp)
+
+target_link_libraries(file_to_stream
+    PRIVATE
+        sparrow-ipc
+        sparrow::sparrow
+        sparrow::json_reader
+)
+
+set_target_properties(file_to_stream
+    PROPERTIES
+        CXX_STANDARD 20
+        CXX_STANDARD_REQUIRED ON
+        CXX_EXTENSIONS OFF
+)
+
+target_include_directories(file_to_stream
+    PRIVATE
+        ${CMAKE_SOURCE_DIR}/include
+        ${CMAKE_BINARY_DIR}/generated
+)
+
+add_dependencies(file_to_stream generate_flatbuffers_headers)
+
+# Create executable for stream_to_file integration test
+add_executable(stream_to_file stream_to_file.cpp)
+
+target_link_libraries(stream_to_file
+    PRIVATE
+        sparrow-ipc
+        sparrow::sparrow
+)
+
+set_target_properties(stream_to_file
+    PROPERTIES
+        CXX_STANDARD 20
+        CXX_STANDARD_REQUIRED ON
+        CXX_EXTENSIONS OFF
+)
+
+target_include_directories(stream_to_file
+    PRIVATE
+        ${CMAKE_SOURCE_DIR}/include
+        ${CMAKE_BINARY_DIR}/generated
+)
+
+add_dependencies(stream_to_file generate_flatbuffers_headers)
+
+# Create test executable for integration tools
+add_executable(test_integration_tools main.cpp test_integration_tools.cpp)
+
+target_link_libraries(test_integration_tools
+    PRIVATE
+        sparrow-ipc
+        sparrow::sparrow
+        sparrow::json_reader
+        doctest::doctest
+        arrow-testing-data
+)
+
+target_compile_definitions(test_integration_tools
+    PRIVATE
+        INTEGRATION_TOOLS_DIR="${CMAKE_CURRENT_BINARY_DIR}"
+)
+
+set_target_properties(test_integration_tools
+    PROPERTIES
+        CXX_STANDARD 20
+        CXX_STANDARD_REQUIRED ON
+        CXX_EXTENSIONS OFF
+)
+
+target_include_directories(test_integration_tools
+    PRIVATE
+        ${CMAKE_SOURCE_DIR}/include
+        ${CMAKE_BINARY_DIR}/generated
+)
+
+add_dependencies(test_integration_tools generate_flatbuffers_headers file_to_stream stream_to_file)
+
+# Register with CTest
+enable_testing()
+add_test(NAME integration_tools_test COMMAND test_integration_tools)
+
+# On Windows, copy required DLLs
+if(WIN32)
+    add_custom_command(
+        TARGET file_to_stream POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow::sparrow>"
+            "$<TARGET_FILE_DIR:file_to_stream>"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow-ipc>"
+            "$<TARGET_FILE_DIR:file_to_stream>"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow::json_reader>"
+            "$<TARGET_FILE_DIR:file_to_stream>"
+        COMMENT "Copying DLLs to file_to_stream executable directory"
+    )
+
+    add_custom_command(
+        TARGET stream_to_file POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow::sparrow>"
+            "$<TARGET_FILE_DIR:stream_to_file>"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow-ipc>"
+            "$<TARGET_FILE_DIR:stream_to_file>"
+        COMMENT "Copying DLLs to stream_to_file executable directory"
+    )
+
+    add_custom_command(
+        TARGET test_integration_tools POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow::sparrow>"
+            "$<TARGET_FILE_DIR:test_integration_tools>"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow-ipc>"
+            "$<TARGET_FILE_DIR:test_integration_tools>"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow::json_reader>"
+            "$<TARGET_FILE_DIR:test_integration_tools>"
+        COMMENT "Copying DLLs to test_integration_tools executable directory"
+    )
+endif()
+
+set_target_properties(file_to_stream stream_to_file test_integration_tools PROPERTIES FOLDER "Integration Tests")