Merge pull request #134 from AppLayerLabs/deps_rev

Automate project dependencies & General optimizations

Author: itamarcps

.github/workflows/c-cpp.yml

@@ -23,11 +23,15 @@ jobs:
       BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
     steps:
       - name: Clone the repository
-        run: git clone https://github.com/${{ github.repository }} ${{ env.TARGET_DIR}} || true
+        run: git clone https://github.com/${{github.repository}} ${{env.TARGET_DIR}} || true
       - name: Sync with remote repository
         run: git -C ${{env.TARGET_DIR}} fetch
       - name: Checkout to current branch
         run: git -C ${{env.TARGET_DIR}} checkout ${{env.BRANCH_NAME}}
+      - name: Set user.email
+        run: git -C ${{env.TARGET_DIR}} config --global user.email "github-actions[bot]@users.noreply.github.com"
+      - name: Set user.name
+        run: git -C ${{env.TARGET_DIR}} config --global user.name "github-actions"
       - name: Update local repository
         run: git -C ${{env.TARGET_DIR}} pull
       - name: Build the container
@@ -36,7 +40,7 @@ jobs:
         run: ./scripts/auto.sh -s bdk stop
       - name: Restart the container
         run: ./scripts/auto.sh -s bdk up
-      - name: Clean previos build (if any)
+      - name: Clean previous build (if there is one)
         run: ./scripts/auto.sh -s bdk exec 'make -C build clean' || true
       - name: Configure MOLD linker
         run: ./scripts/auto.sh -s bdk exec 'cmake -S . -B build

CMakeLists.txt

@@ -16,6 +16,8 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
   cmake_policy(SET CMP0135 NEW)
 endif()
 
+# TODO: avoid FindBoost deprecation message in CMake 3.30+ (cmake --help-policy CMP0167)
+
 # Project data
 project(bdk VERSION 0.2.0 DESCRIPTION "AppLayer Blockchain Development Kit")
 set(CMAKE_CXX_STANDARD 23)
@@ -59,8 +61,8 @@ set(BUILD_DISCOVERY ON CACHE BOOL "Build helper discovery node program")
 set(BUILD_TOOLS OFF CACHE BOOL "Build tools related to subnet")
 set(BUILD_TESTNET OFF CACHE BOOL "Build the project for testnet")
 set(BUILD_BENCHMARK OFF CACHE BOOL "Build with the benchmark tests")
-set(USE_LINT OFF CACHE BOOL "Run linter on compile (clang-tidy)")
 set(BUILD_VARIABLES_TESTS ON CACHE BOOL "Build tests for SafeVar (Contract variables)")
+set(USE_LINT OFF CACHE BOOL "Run linter on compile (clang-tidy)")
 if(USE_LINT)
   set(CMAKE_CXX_CLANG_TIDY "clang-tidy;-header-filter=.;-checks=-*,abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,hicpp-*,misc-*,modernize-*,performance-*,portability-*,readability-*")
 endif()
@@ -79,6 +81,9 @@ message(STATUS "Find libs with suffix: ${CMAKE_FIND_LIBRARY_SUFFIXES}")
 message("Building tests: ${BUILD_TESTS}")
 message("Building Discovery Node: ${BUILD_DISCOVERY}")
 message("Building tools: ${BUILD_TOOLS}")
+message("Building testnet: ${BUILD_TESTNET}")
+message("Building benchmark tests: ${BUILD_BENCHMARK}")
+message("Building SafeVar tests: ${BUILD_VARIABLES_TESTS}")
 message("Using lint: ${USE_LINT}")
 
 cable_add_buildinfo_library(PROJECT_NAME bdk)
@@ -89,21 +94,22 @@ set(OPENSSL_USE_STATIC_LIBS ON)
 set(Protobuf_USE_STATIC_LIBS ON)
 
 # Find system packages (built-in)
-find_package(Threads)
 find_package(Boost 1.83.0 REQUIRED COMPONENTS chrono filesystem program_options system thread nowide)
 find_package(OpenSSL 1.1.1 REQUIRED)
-find_package(ZLIB REQUIRED)
+find_package(Protobuf REQUIRED) # TODO: not used yet but will be, keep it for now
+find_package(Threads)
 
 # Find system packages (custom)
+find_package(Cares REQUIRED) # TODO: not used yet but will be, keep it for now
 find_package(CryptoPP 8.2.0 REQUIRED)
+find_package(Ethash REQUIRED)
+find_package(Evmc REQUIRED)
+find_package(Evmone REQUIRED)
+find_package(GRPC REQUIRED) # TODO: not used yet but will be, keep it for now
+find_package(Keccak REQUIRED)
 find_package(Scrypt REQUIRED)
-
-# Add external modules
-include(cmake/ProjectEthash.cmake)        # Ethash
-include(cmake/ProjectSecp256k1.cmake)     # Bitcoin core fast implementation
-include(cmake/ProjectSpeedb.cmake)        # Speedb (Level/RocksDB drop-in replacement)
-include(cmake/ProjectEVMOne.cmake)        # EVMOne (EVMOne + EVMC)
-
+find_package(Secp256k1 REQUIRED)
+find_package(Speedb REQUIRED)
 
 # Add catch2 as a library
 add_library(catch2
@@ -141,21 +147,21 @@ add_subdirectory(src/utils)
 add_subdirectory(tests)
 
 add_library(bdk_lib STATIC
-  ${UTILS_HEADERS}
-  ${UTILS_SOURCES}
-  ${CONTRACT_HEADERS}
-  ${CONTRACT_SOURCES}
-  ${CORE_HEADERS}
-  ${CORE_SOURCES}
-  ${NET_HEADERS}
-  ${NET_SOURCES}
+  ${UTILS_HEADERS} ${UTILS_SOURCES} ${CONTRACT_HEADERS} ${CONTRACT_SOURCES}
+  ${CORE_HEADERS} ${CORE_SOURCES} ${NET_HEADERS} ${NET_SOURCES}
 )
 
-target_include_directories(bdk_lib PRIVATE ${CMAKE_SOURCE_DIR}/include ${OPENSSL_INCLUDE_DIR})
+target_include_directories(bdk_lib PRIVATE
+  ${CMAKE_SOURCE_DIR}/include ${OPENSSL_INCLUDE_DIR} ${ETHASH_INCLUDE_DIR} ${KECCAK_INCLUDE_DIR}
+  ${EVMC_INCLUDE_DIR} ${EVMONE_INCLUDE_DIR} ${SPEEDB_INCLUDE_DIR} ${SECP256K1_INCLUDE_DIR}
+)
 
-target_link_libraries(bdk_lib PRIVATE EvmcInstructions EvmcLoader EvmcTooling Evmone
-  ${CRYPTOPP_LIBRARIES} ${SCRYPT_LIBRARY} Secp256k1 Ethash ${ETHASH_BYPRODUCTS}
-  Speedb ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES})
+target_link_libraries(bdk_lib PRIVATE
+  ${EVMC_INSTRUCTIONS_LIBRARY} ${EVMC_LOADER_LIBRARY} ${EVMONE_LIBRARY}
+  ${CRYPTOPP_LIBRARIES} ${SCRYPT_LIBRARY} ${SECP256K1_LIBRARY}
+  ${ETHASH_LIBRARY} ${KECCAK_LIBRARY} ${SPEEDB_LIBRARY}
+  ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES}
+)
 
 add_subdirectory(src/bins)
 

Dockerfile

@@ -8,40 +8,18 @@ FROM debian:trixie
 # Update the system
 RUN apt-get update && apt-get upgrade -y
 
-# Install dependencies
-RUN apt-get install -y \
-    build-essential \
-    cmake \
-    tmux \
-    clang-tidy \
-    autoconf \
-    libtool \
-    pkg-config \
-    libabsl-dev \
-    libboost-all-dev \
-    libc-ares-dev \
-    libcrypto++-dev \
-    libgrpc-dev \
-    libgrpc++-dev \
-    librocksdb-dev \
-    libscrypt-dev \
-    libsnappy-dev \
-    libssl-dev \
-    zlib1g-dev \
-    openssl \
-    protobuf-compiler \
-    protobuf-compiler-grpc \
-    nano \
-    vim \
-    unison \
-    git
-
 # Set the working directory in the Docker container
 WORKDIR /bdk-cpp
 
 # Copy the local folder to the container
 COPY . /bdk-cpp
 
+# Install Docker-specific dependencies
+RUN apt-get -y install nano vim unison curl jq unzip gcovr
+
+# Install dependencies
+RUN bash /bdk-cpp/scripts/deps.sh --install
+
 # Create the synchronized directory
 RUN mkdir /bdk-volume
 

README.md

@@ -35,28 +35,40 @@ The project has a Dockerfile at the root of the repository that will build the p
 * Build the image locally with `docker build -t bdk-cpp-dev:latest .` (if using Linux or Mac, run as `sudo`)
   * This will build the image and tag it as `bdk-cpp-dev:latest` - you can change the tag to whatever you want, but remember to change it on the next step
 * Run the container (you will be logged in as root):
-  * **For Linux/Mac**: `sudo docker run -it -v $(pwd):/bdk-volume -p 8080-8099:8080-8099 -p 8110-8111:8110-8111 bdk-cpp-dev:latest`
-  * **For Windows**: `docker run -it -v %cd%:/bdk-volume -p 8080-8099:8080-8099 -p 8110-8111:8110-8111 bdk-cpp-dev:latest`
+  * **For Linux/Mac**: `sudo docker run -it --name bdk-cpp -v $(pwd):/bdk-volume -p 8080-8099:8080-8099 -p 8110-8111:8110-8111 bdk-cpp-dev:latest`
+  * **For Windows**: `docker run -it --name bdk-cpp -v %cd%:/bdk-volume -p 8080-8099:8080-8099 -p 8110-8111:8110-8111 bdk-cpp-dev:latest`
 
 Remember that we are using our local repo as a volume, so every change in the local folder will be reflected to the container in real time, and vice-versa.
 
 Also, you can integrate the container with your favorite IDE or editor, e.g. [VSCode + Docker extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker).
 
 ## Developing manually
 
-Install the following dependencies on your system:
-
-* **GCC** with support for **C++23** or higher
-* **CMake 3.19.0** or higher
-* **Boost 1.83** or higher (components: *chrono, filesystem, program-options, system, thread, nowide*)
-* **OpenSSL 1.1.1**
-* **CryptoPP 8.2.0** or higher
-* **libscrypt**
-* **zlib**
-* **libsnappy** for database compression
-* **tmux** (for deploying)
-* (optional) **clang-tidy** for linting
-* (optional) **mold** for faster/more efficient linking
+You will need the following dependencies installed locally on your system:
+
+* *Toolchain binaries*:
+  * **git**
+  * **GCC** with support for **C++23** or higher
+  * **Make**
+  * **CMake 3.19.0** or higher
+  * **Protobuf** (protoc + grpc_cpp_plugin)
+  * **tmux** (for deploying)
+  * (optional) **ninja** if you prefer it over make
+  * (optional) **mold** if you prefer it over ld
+  * (optional) **doxygen** for generating docs
+  * (optional) **clang-tidy** for linting
+* *Libraries*:
+  * **Boost 1.83** or higher (components: *chrono, filesystem, program-options, system, thread, nowide*)
+  * **OpenSSL 1.1.1** / **libssl 1.1.1** or higher
+  * **libzstd**
+  * **CryptoPP 8.2.0** or higher
+  * **libscrypt**
+  * **libc-ares**
+  * **gRPC** (libgrpc and libgrpc++)
+  * **secp256k1**
+  * **ethash** + **keccak**
+  * **EVMOne** + **EVMC**
+  * **Speedb**
 
 The versions of those dependencies should suffice out-of-the-box for at least the following distros (or greater, including their derivatives):
 
@@ -66,21 +78,23 @@ The versions of those dependencies should suffice out-of-the-box for at least th
 * **Fedora 40**
 * Any rolling release distro from around **May 2024** onwards (check their repos to be sure)
 
-For older distros, you may need to compile some dependencies from source (specifically CMake, Boost and/or GCC). Make sure to either uninstall them from the system first to prevent any version conflicts, or use a workaround like e.g. Debian's `update-alternatives` or similar.
+There is a script called `scripts/deps.sh` which you can use to check if you have those dependencies installed (`deps.sh --check`), and install them in case you don't (`deps.sh --install`). The script expects dependencies to be installed either on `/usr` or `/usr/local`, giving preference to the latter if it finds anything there (so you can use a higher version of a dependency while still keeping your distro's default one).
 
-Specifically for GCC, make sure to also export the right paths for compilation in your environment in case you're using a self-compiled build. For example, in a Linux system, put something like this in your `~/.bashrc` file, changing the version accordingly to whichever one you have installed:
+**Please note that installing most dependencies through the script only works on APT-based distros** (Debian, Ubuntu and derivatives) - you can still check the dependencies on any distro and install the few ones labeled as "external" (those are fetched through `git`), but if you're on a distro with another package manager and/or a distro older than one of the minimum ones listed above, you're on your own.
 
-```bash
-PATH=/usr/local/gcc-14.1.0/bin:$PATH
-LD_LIBRARY_PATH=/usr/local/gcc-14.1.0/lib64:$LD_LIBRARY_PATH
-```
+### GCC-specific tips
 
-#### One-liners
+For Debian specifically, you can (and should) use `update-alternatives` to register and set your GCC version to a more up-to-date build if required.
 
-* For APT-based distros:
+If you're using a self-compiled GCC build out of the system path (e.g. `--prefix=/usr/local/gcc-X.Y.Z` instead of `--prefix=/usr/local`), don't forget to export its installation paths in your `PATH` and `LD_LIBRARY_PATH` env vars (to prevent e.g. "version `GLIBCXX_...'/`CXXABI_...` not found" errors). Put something like this in your `~/.bashrc` file for example, changing the version accordingly to whichever one you have installed:
 
 ```bash
-sudo apt install build-essential cmake tmux clang-tidy autoconf libtool pkg-config libabsl-dev libboost-all-dev libc-ares-dev libcrypto++-dev libgrpc-dev libgrpc++-dev librocksdb-dev libscrypt-dev libsnappy-dev libssl-dev zlib1g-dev openssl protobuf-compiler protobuf-compiler-grpc unison git
+# For GCC in /usr/local
+export LD_LIBRARY_PATH=/usr/local/lib64:$LD_LIBRARY_PATH
+
+# For self-contained GCC outside /usr/local
+export PATH=/usr/local/gcc-14.2.0/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/gcc-14.2.0/lib64:$LD_LIBRARY_PATH
 ```
 
 ## Documentation