Merge branch 'main' into upgrade-latest-psycopg-version

Author: quinna-h

.github/workflows/build_deploy.yml

@@ -111,6 +111,17 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
+
+      - name: Enable sccache
+        uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad  # 0.0.9
+
+      - name: Add addtional GHA cache-related env vars
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
+            core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
+
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: source-dist
@@ -128,6 +139,10 @@ jobs:
       - name: Install source package
         env:
           CMAKE_BUILD_PARALLEL_LEVEL: 12
+          CARGO_BUILD_JOBS: 12
+          SCCACHE_GHA_ENABLED: true
+          SCCACHE_CACHE_SIZE: 1G
+          DD_USE_SCCACHE: 1
         run: pip install dist/*.tar.gz
 
       - name: Test the source package

.github/workflows/build_python_3.yml

@@ -83,52 +83,26 @@ jobs:
       fail-fast: false
       matrix:
         include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
-    env:
-      SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
-      CIBW_SKIP: ${{ inputs.cibw_skip }}
-      CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
-      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-      CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
-      CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
-      CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
-      CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
-      CIBW_BEFORE_ALL_LINUX: |
-        if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
-          curl -sSf https://sh.rustup.rs | sh -s -- -y;
-        fi
-      CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
-      # `platform.mac_ver()` reports incorrect MacOS version at 11.0
-      # See: https://stackoverflow.com/a/65402241
-      CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      CIBW_ENVIRONMENT_WINDOWS: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      # cibuildwheel repair will copy anything's under /output directory from the
-      # build container to the host machine. This is a bit hacky way, but seems
-      # to be the only way getting debug symbols out from the container while
-      # we don't mess up with RECORD file.
-      CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
-        mkdir -p /output/debugwheelhouse &&
-        python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
-        python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
-        mkdir ./tempwheelhouse &&
-        unzip -l {wheel} | grep '\.so' &&
-        auditwheel repair -w ./tempwheelhouse {wheel} &&
-        mv ./tempwheelhouse/*.whl {dest_dir} &&
-        rm -rf ./tempwheelhouse
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
-        mkdir -p ./debugwheelhouse &&
-        python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
-        python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
-        MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
-      CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
-      CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
-
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           fetch-depth: 0
 
+      - name: Enable sccache
+        if: runner.os != 'Windows'
+        uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad  # 0.0.9
+        with:
+          disable_annotations: true
+
+      - name: Add addtional GHA cache-related env vars
+        uses: actions/github-script@v7
+        if: runner.os != 'Windows'
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
+            core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
+
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         name: Install Python
         with:
@@ -144,7 +118,76 @@ jobs:
         uses: pypa/cibuildwheel@c923d83ad9c1bc00211c5041d0c3f73294ff88f6 # v3.1.4
         with:
           only: ${{ matrix.only }}
-
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
+          CIBW_SKIP: ${{ inputs.cibw_skip }}
+          CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
+          CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
+          CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
+          CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
+          CIBW_BEFORE_ALL_LINUX: |
+            if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
+              curl -sSf https://sh.rustup.rs | sh -s -- -y;
+            fi
+          CIBW_ENVIRONMENT_LINUX: >
+            PATH=$HOME/.cargo/bin:$PATH
+            CARGO_BUILD_JOBS=24
+            CMAKE_BUILD_PARALLEL_LEVEL=24
+            CMAKE_ARGS="-DNATIVE_TESTING=OFF"
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
+            SCCACHE_GHA_ENABLED=true
+            SCCACHE_DIR=/host/${{ env.SCCACHE_DIR }}
+            SCCACHE_PATH=/host/${{ env.SCCACHE_PATH }}
+            SCCACHE_CACHE_SIZE=1G
+            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
+            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
+            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
+            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
+            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
+            DD_USE_SCCACHE=1
+          # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
+          # `platform.mac_ver()` reports incorrect MacOS version at 11.0
+          # See: https://stackoverflow.com/a/65402241
+          CIBW_ENVIRONMENT_MACOS: >
+            CMAKE_BUILD_PARALLEL_LEVEL=24
+            CARGO_BUILD_JOBS=24
+            SYSTEM_VERSION_COMPAT=0
+            CMAKE_ARGS="-DNATIVE_TESTING=OFF"
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
+            SCCACHE_GHA_ENABLED=true
+            SCCACHE_CACHE_SIZE=1G
+            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
+            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
+            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
+            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
+            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
+            DD_USE_SCCACHE=1
+          CIBW_ENVIRONMENT_WINDOWS: >
+            SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
+          # cibuildwheel repair will copy anything's under /output directory from the
+          # build container to the host machine. This is a bit hacky way, but seems
+          # to be the only way getting debug symbols out from the container while
+          # we don't mess up with RECORD file.
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
+            mkdir -p /output/debugwheelhouse &&
+            python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
+            python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
+            mkdir ./tempwheelhouse &&
+            unzip -l {wheel} | grep '\.so' &&
+            auditwheel repair -w ./tempwheelhouse {wheel} &&
+            mv ./tempwheelhouse/*.whl {dest_dir} &&
+            rm -rf ./tempwheelhouse
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
+            mkdir -p ./debugwheelhouse &&
+            python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
+            python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
+            MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
+          CIBW_BEFORE_TEST_LINUX: /host/${{ env.SCCACHE_PATH }} --show-stats
+          CIBW_BEFORE_TEST_MACOS: ${{ env.SCCACHE_PATH }} --show-stats
+          CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
       - name: Validate wheel RECORD files
         shell: bash
         run: |

.riot/requirements/16ebde6.txt

@@ -54,6 +54,8 @@ add_library(
     src/code_provenance_interface.cpp
     src/ddup_interface.cpp
     src/profile.cpp
+    src/profile_borrow.cpp
+    src/profiler_stats.cpp
     src/sample.cpp
     src/sample_manager.cpp
     src/static_sample_pool.cpp

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -1,14 +1,13 @@
 #pragma once
 
-#include <stddef.h>
-#include <stdint.h>
+#include <cstdint>
 #include <string_view>
 #include <unordered_map>
 
 // Forward decl of the return pointer
 namespace Datadog {
 class Sample;
-}
+} // namespace Datadog
 
 #ifdef __cplusplus
 extern "C"
@@ -68,6 +67,10 @@ extern "C"
                          int64_t line);
     void ddup_push_absolute_ns(Datadog::Sample* sample, int64_t timestamp_ns);
     void ddup_push_monotonic_ns(Datadog::Sample* sample, int64_t monotonic_ns);
+
+    void ddup_increment_sampling_event_count();
+    void ddup_increment_sample_count();
+
     void ddup_flush_sample(Datadog::Sample* sample);
     // Stack v2 specific flush, which reverses the locations
     void ddup_flush_sample_v2(Datadog::Sample* sample);

ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp

@@ -1,13 +1,11 @@
 #pragma once
 
 #include "constants.hpp"
+#include "profiler_stats.hpp"
 #include "types.hpp"
 
 #include <atomic>
-#include <memory>
 #include <mutex>
-#include <string>
-#include <string_view>
 #include <vector>
 
 extern "C"
@@ -17,9 +15,13 @@ extern "C"
 
 namespace Datadog {
 
+class ProfileBorrow;
+
 // Serves to collect individual samples, as well as lengthen the scope of string data
 class Profile
 {
+    friend class ProfileBorrow;
+
   private:
     // Serialization for static state
     // - string table
@@ -45,6 +47,12 @@ class Profile
     // cannot be used until it's initialized by libdatadog
     ddog_prof_Profile cur_profile{};
 
+    Datadog::ProfilerStats profiler_stats{};
+
+    // Internal access methods - not for direct use
+    ddog_prof_Profile& profile_borrow_internal();
+    void profile_release();
+
   public:
     // State management
     void one_time_init(SampleType type, unsigned int _max_nframes);
@@ -53,8 +61,8 @@ class Profile
 
     // Getters
     size_t get_sample_type_length();
-    ddog_prof_Profile& profile_borrow();
-    void profile_release();
+
+    ProfileBorrow borrow();
 
     // constref getters
     const ValueIndex& val();

ddtrace/internal/datadog/profiling/dd_wrapper/include/profile.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "profile.hpp"
+
+namespace Datadog {
+
+// Forward declaration
+class Profile;
+
+// RAII wrapper for borrowing both profile and stats under a single lock
+class ProfileBorrow
+{
+  private:
+    Profile* profile_ptr;
+
+  public:
+    explicit ProfileBorrow(Profile& profile);
+    ~ProfileBorrow();
+
+    // Disable copy
+    ProfileBorrow(const ProfileBorrow&) = delete;
+    ProfileBorrow& operator=(const ProfileBorrow&) = delete;
+
+    // Enable move
+    ProfileBorrow(ProfileBorrow&& other) noexcept;
+    ProfileBorrow& operator=(ProfileBorrow&& other) noexcept;
+
+    // Accessors
+    ddog_prof_Profile& profile();
+    ProfilerStats& stats();
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/profile_borrow.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cstddef>
+
+#include <string>
+#include <string_view>
+
+namespace Datadog {
+
+/*
+ProfilerStats holds statistics around Profiling to be sent along
+with the actual Profiles.
+
+None of its methods are thread-safe and it should typically used with
+a mutex to protect access to the data.
+*/
+class ProfilerStats
+{
+  private:
+    std::string internal_metadata_json;
+
+    // Number of samples collected (one per thread)
+    size_t sample_count = 0;
+
+    // Number of sampling events (one per collection cycle)
+    size_t sampling_event_count = 0;
+
+  public:
+    ProfilerStats() = default;
+    ~ProfilerStats() = default;
+
+    void increment_sample_count(size_t k_sample_count = 1);
+    size_t get_sample_count();
+
+    void increment_sampling_event_count(size_t k_sampling_event_count = 1);
+    size_t get_sampling_event_count();
+
+    // Returns a JSON string containing relevant Profiler Stats to be included
+    // in the libdatadog payload.
+    // The function returned a string_view to a statically allocated string that
+    // is updated every time the function is called.
+    std::string_view get_internal_metadata_json();
+
+    void reset_state();
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp

@@ -2,6 +2,7 @@
 
 #include "libdatadog_helpers.hpp"
 #include "profile.hpp"
+#include "profile_borrow.hpp"
 #include "types.hpp"
 
 #include <string>
@@ -134,8 +135,7 @@ class Sample
     // Flushes the current buffer, clearing it
     bool flush_sample(bool reverse_locations = false);
 
-    static ddog_prof_Profile& profile_borrow();
-    static void profile_release();
+    static ProfileBorrow profile_borrow();
     static void postfork_child();
     Sample(SampleType _type_mask, unsigned int _max_nframes);