BREAKING CHANGE: buffers exposed as binary, as a UTF-8 string (#94)

Author: kafka1991

.clang-format

@@ -0,0 +1,36 @@
+---
+BasedOnStyle: Microsoft
+AlignAfterOpenBracket: AlwaysBreak
+AlignTrailingComments: 'true'
+AllowAllArgumentsOnNextLine: 'true'
+AllowAllConstructorInitializersOnNextLine: 'true'
+AllowAllParametersOfDeclarationOnNextLine: 'true'
+AllowShortBlocksOnASingleLine: 'false'
+AllowShortCaseLabelsOnASingleLine: 'false'
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: Inline
+AllowShortLoopsOnASingleLine: 'false'
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: 'true'
+AlwaysBreakTemplateDeclarations: 'Yes'
+BinPackArguments: 'false'
+BinPackParameters: 'false'
+BreakBeforeTernaryOperators: 'true'
+BreakConstructorInitializers: BeforeComma
+BreakInheritanceList: BeforeComma
+ColumnLimit: '80'
+CompactNamespaces: 'true'
+FixNamespaceComments: 'false'
+IndentWidth: '4'
+PointerAlignment: Left
+SortIncludes: 'false'
+SortUsingDeclarations: 'true'
+SpaceAfterTemplateKeyword: 'true'
+UseTab: Never
+IndentPPDirectives: AfterHash
+AccessModifierOffset: -4
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterExternBlock: false
+...

.gitignore

@@ -1,5 +1,6 @@
 target
 build
+build_CXX20
 cmake-build-debug
 cmake-build-release
 .vs

CMakeLists.txt

@@ -16,7 +16,13 @@ set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_C_EXTENSIONS OFF)
 
-set(CMAKE_CXX_STANDARD 17)
+set(MINIMUM_REQUIRED_CXX_STANDARD 17)
+
+if((NOT CMAKE_CXX_STANDARD) OR (CMAKE_CXX_STANDARD LESS MINIMUM_REQUIRED_CXX_STANDARD))
+    set(CMAKE_CXX_STANDARD ${MINIMUM_REQUIRED_CXX_STANDARD})
+endif()
+message(STATUS "C++ standard: ${CMAKE_CXX_STANDARD}")
+
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 

ci/compile.yaml

@@ -12,3 +12,11 @@ steps:
     env:
       JAVA_HOME: $(JAVA_HOME_11_X64)
     displayName: "Make"
+  - script: cmake -S . -B build_CXX20 -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON -DCMAKE_CXX_STANDARD=20
+    env:
+      JAVA_HOME: $(JAVA_HOME_11_X64)
+    displayName: "Build Makefile with CMake"
+  - script: cmake --build build_CXX20 --config Release
+    env:
+      JAVA_HOME: $(JAVA_HOME_11_X64)
+    displayName: "Make"

ci/format_cpp.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+"""
+Format all the C and C++ code using `clang-format`.
+If --check is passed, check for formatting issues instead of modifying files.
+"""
+
+import sys
+sys.dont_write_bytecode = True
+import subprocess
+
+FILES = [
+    'include/questdb/ingress/line_sender.h',
+    'include/questdb/ingress/line_sender.hpp',
+    'cpp_test/build_env.h',
+    'cpp_test/mock_server.hpp',
+    'cpp_test/mock_server.cpp',
+    'cpp_test/test_line_sender.cpp',
+]
+
+if __name__ == '__main__':
+    check_mode = '--check' in sys.argv
+    command = [
+        'clang-format',
+        '--style=file',
+        '--dry-run' if check_mode else '-i'
+        ] + FILES
+    subprocess.check_call(command)

ci/run_all_tests.py

@@ -31,18 +31,22 @@ def run_cmd(*args, cwd=None):
         sys.stderr.write(f'Command {args!r} failed with return code {cpe.returncode}.\n')
         sys.exit(cpe.returncode)
 
-
 def main():
     build_dir = pathlib.Path('build')
     exe_suffix = '.exe' if platform.system() == 'Windows' else ''
     test_line_sender_path = next(iter(
         build_dir.glob(f'**/test_line_sender{exe_suffix}')))
+    build_cxx20_dir = pathlib.Path('build_CXX20')
+    test_line_sender_path_CXX20 = next(iter(
+        build_cxx20_dir.glob(f'**/test_line_sender{exe_suffix}'))) 
+
     system_test_path = pathlib.Path('system_test') / 'test.py'
     qdb_v = '8.1.0'  # The version of QuestDB we'll test against.
 
     run_cmd('cargo', 'test', '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', '--all-features', '--', '--nocapture', cwd='questdb-rs')
     run_cmd(str(test_line_sender_path))
+    run_cmd(str(test_line_sender_path_CXX20))
     run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v')
 
 

ci/run_tests_pipeline.yaml

@@ -62,17 +62,18 @@ stages:
             inputs:
               pathToPublish: ./build
             displayName: "Publish build directory"
-      - job: CargoFmtAndClippy
+      - job: FormatAndLinting
         displayName: "cargo fmt and clippy"
         pool:
           vmImage: 'ubuntu-latest'
         timeoutInMinutes: 10
         steps:
           - checkout: self
           - script: |
+              apt install clang-format
               rustup component add clippy
               rustup component add rustfmt
-            displayName: "Install clippy and rustfmt"
+            displayName: "Install clang-format, clippy and rustfmt"
           - script: |
               cd questdb-rs
               cargo fmt --all -- --check
@@ -89,6 +90,9 @@ stages:
               cd questdb-rs-ffi
               cargo clippy --all-targets --all-features -- -D warnings
             displayName: "questdb-rs-ffi: clippy"
+          - script: |
+              python3 ci/format_cpp.py --check
+            displayName: "C/C++ clang-format"
           - script: |
               cd system_test
               cd tls_proxy

cpp_test/mock_server.cpp

@@ -27,27 +27,27 @@
 #include <string.h>
 
 #if defined(PLATFORM_UNIX)
-#include <fcntl.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <unistd.h>
+#    include <fcntl.h>
+#    include <arpa/inet.h>
+#    include <netdb.h>
+#    include <netinet/in.h>
+#    include <netinet/tcp.h>
+#    include <sys/types.h>
+#    include <sys/socket.h>
+#    include <unistd.h>
 #elif defined(PLATFORM_WINDOWS)
-#include <winsock2.h>
-#include <ws2tcpip.h>
+#    include <winsock2.h>
+#    include <ws2tcpip.h>
 #endif
 
 #if defined(PLATFORM_UNIX)
-#define CLOSESOCKET ::close
+#    define CLOSESOCKET ::close
 typedef const void* setsockopt_arg_t;
-#ifndef INVALID_SOCKET
-#define INVALID_SOCKET -1
-#endif
+#    ifndef INVALID_SOCKET
+#        define INVALID_SOCKET -1
+#    endif
 #elif defined(PLATFORM_WINDOWS)
-#define CLOSESOCKET ::closesocket
+#    define CLOSESOCKET ::closesocket
 typedef const char* setsockopt_arg_t;
 typedef long suseconds_t;
 #endif
@@ -108,8 +108,8 @@ mock_server::mock_server()
             _listen_fd,
             SOL_SOCKET,
             SO_REUSEADDR,
-            static_cast<setsockopt_arg_t>(static_cast<const void*>(
-                &reuse_addr)),
+            static_cast<setsockopt_arg_t>(
+                static_cast<const void*>(&reuse_addr)),
             sizeof(reuse_addr)) != 0)
     {
 #if defined(PLATFORM_UNIX)
@@ -126,10 +126,8 @@ mock_server::mock_server()
     listen_addr.sin_family = AF_INET;
     listen_addr.sin_addr.s_addr = INADDR_ANY;
     listen_addr.sin_port = htons(0);
-    if (bind(
-            _listen_fd,
-            (const sockaddr *)&listen_addr,
-            sizeof(listen_addr)) == -1)
+    if (bind(_listen_fd, (const sockaddr*)&listen_addr, sizeof(listen_addr)) ==
+        -1)
         throw std::runtime_error{"Bad `bind()`."};
 
     if (listen(_listen_fd, 1) == -1)
@@ -139,9 +137,7 @@ mock_server::mock_server()
     memset(&resolved_addr, 0, sizeof(resolved_addr));
     socklen_t resolved_addr_len = sizeof(resolved_addr);
     if (getsockname(
-            _listen_fd,
-            (sockaddr *)&resolved_addr,
-            &resolved_addr_len) == -1)
+            _listen_fd, (sockaddr*)&resolved_addr, &resolved_addr_len) == -1)
         throw std::runtime_error{"Bad `getsockname()`."};
     _port = ntohs(resolved_addr.sin_port);
 }
@@ -150,10 +146,7 @@ void mock_server::accept()
 {
     sockaddr_in remote_addr;
     socklen_t remote_addr_len = sizeof(remote_addr);
-    _conn_fd = ::accept(
-        _listen_fd,
-        (sockaddr *)&remote_addr,
-        &remote_addr_len);
+    _conn_fd = ::accept(_listen_fd, (sockaddr*)&remote_addr, &remote_addr_len);
     if (_conn_fd == INVALID_SOCKET)
         throw std::runtime_error{"Bad `accept()`."};
 #if defined(PLATFORM_UNIX)
@@ -169,7 +162,7 @@ bool mock_server::wait_for_data(std::optional<double> wait_timeout_sec)
     fd_set read_set;
     FD_ZERO(&read_set);
     FD_SET(_conn_fd, &read_set);
-    timeval* timeout_ptr = nullptr;  // nullptr blocks indefinitely.
+    timeval* timeout_ptr = nullptr; // nullptr blocks indefinitely.
     timeval timeout;
     if (wait_timeout_sec)
     {
@@ -178,9 +171,8 @@ bool mock_server::wait_for_data(std::optional<double> wait_timeout_sec)
 #elif defined(PLATFORM_WINDOWS)
         const long secs = static_cast<long>(*wait_timeout_sec);
 #endif
-        const suseconds_t usec =
-            static_cast<suseconds_t>(
-                1000000.0 * (*wait_timeout_sec - static_cast<double>(secs)));
+        const suseconds_t usec = static_cast<suseconds_t>(
+            1000000.0 * (*wait_timeout_sec - static_cast<double>(secs)));
         timeout = timeval{secs, usec};
         timeout_ptr = &timeout;
     }
@@ -209,11 +201,8 @@ size_t mock_server::recv(double wait_timeout_sec)
     for (;;)
     {
         wait_for_data();
-        sock_ssize_t count = ::recv(
-            _conn_fd,
-            &chunk[0],
-            static_cast<sock_len_t>(chunk_len),
-            0);
+        sock_ssize_t count =
+            ::recv(_conn_fd, &chunk[0], static_cast<sock_len_t>(chunk_len), 0);
         if (count == -1)
             throw std::runtime_error{"Bad `recv()`."};
         const size_t u_count = static_cast<size_t>(count);
@@ -266,4 +255,4 @@ mock_server::~mock_server()
 #endif
 }
 
-}
+} // namespace questdb::ingress::test

cpp_test/mock_server.hpp

@@ -35,7 +35,7 @@
 #if defined(PLATFORM_UNIX)
 typedef int socketfd_t;
 #elif defined(PLATFORM_WINDOWS)
-#include <winsock2.h>
+#    include <winsock2.h>
 typedef SOCKET socketfd_t;
 #endif
 
@@ -45,17 +45,20 @@ namespace questdb::ingress::test
 /**
  * Bug-ridden mock server to handle line requests.
  * YMMV, but should be just good enough for testing.
-*/
+ */
 class mock_server
 {
 public:
     mock_server();
 
-    uint16_t port() const { return _port; }
+    uint16_t port() const
+    {
+        return _port;
+    }
 
     void accept();
 
-    size_t recv(double wait_timeout_sec=0.1);
+    size_t recv(double wait_timeout_sec = 0.1);
 
     const std::vector<std::string>& msgs() const
     {
@@ -75,4 +78,4 @@ class mock_server
     std::vector<std::string> _msgs;
 };
 
-}
+} // namespace questdb::ingress::test