From d0b375f367509da1abe9e315bd0459c63b340333 Mon Sep 17 00:00:00 2001 From: dudik Date: Sat, 15 Nov 2025 19:52:15 +0100 Subject: [PATCH] Introduce MemoryPressureMonitor to detect high memory pressure --- .../Debugger/Caching/DefaultMemoryChecker.cs | 50 +- .../Debugger/RateLimiting/IGCInfoProvider.cs | 34 + .../RateLimiting/IHighResolutionClock.cs | 18 + .../RateLimiting/IMemoryPressureMonitor.cs | 46 + .../RateLimiting/ISamplerScheduler.cs | 24 + .../RateLimiting/MemoryPressureConfig.cs | 62 ++ .../RateLimiting/MemoryPressureMonitor.cs | 290 ++++++ .../Debugger/RateLimiting/SystemClock.cs | 24 + .../RateLimiting/SystemGCInfoProvider.cs | 85 ++ .../RateLimiting/WindowsMemoryInfo.cs | 72 ++ .../Debugger/RateLimiting/FakeClock.cs | 42 + .../RateLimiting/FakeGCInfoProvider.cs | 83 ++ .../MemoryPressureMonitorTests.cs | 906 ++++++++++++++++++ .../Debugger/RateLimiting/TestScheduler.cs | 71 ++ 14 files changed, 1759 insertions(+), 48 deletions(-) create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/IGCInfoProvider.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/IHighResolutionClock.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/IMemoryPressureMonitor.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/ISamplerScheduler.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureConfig.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureMonitor.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemClock.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemGCInfoProvider.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/WindowsMemoryInfo.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeClock.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeGCInfoProvider.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/MemoryPressureMonitorTests.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/TestScheduler.cs diff --git a/tracer/src/Datadog.Trace/Debugger/Caching/DefaultMemoryChecker.cs b/tracer/src/Datadog.Trace/Debugger/Caching/DefaultMemoryChecker.cs index a635c9f40253..075ecef1c9b4 100644 --- a/tracer/src/Datadog.Trace/Debugger/Caching/DefaultMemoryChecker.cs +++ b/tracer/src/Datadog.Trace/Debugger/Caching/DefaultMemoryChecker.cs @@ -6,9 +6,7 @@ #nullable enable using System; -using System.Linq; -using System.Runtime.InteropServices; -using System.Threading; +using Datadog.Trace.Debugger.RateLimiting; using Datadog.Trace.Logging; using Datadog.Trace.VendoredMicrosoftCode.System; @@ -29,15 +27,10 @@ private DefaultMemoryChecker() public bool IsLowResourceEnvironment { get; } - [return: MarshalAs(UnmanagedType.Bool)] - [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] - private static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer); - private bool CheckLowResourceEnvironment() { try { - Logger.Debug("Checking if environment is low on resources"); // Check if we're using more than 75% of available memory or there is less than 1GB of RAM available. return IsLowResourceEnvironmentGc() || IsLowResourceEnvironmentSystem(); } @@ -76,7 +69,7 @@ internal bool CheckWindowsMemory() { try { - if (MEMORYSTATUSEX.GetAvailablePhysicalMemory(out var availableMemory)) + if (WindowsMemoryInfo.TryGetAvailablePhysicalMemory(out var availableMemory)) { // If less than 1GB of RAM is available, consider it a low-resource environment return availableMemory < 1_073_741_824; // 1 GB in bytes @@ -151,43 +144,4 @@ protected virtual Datadog.Trace.VendoredMicrosoftCode.System.ReadOnlySpan return value.Slice(0, spaceIndex); } - - // Windows API for memory information - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] - private class MEMORYSTATUSEX - { -#pragma warning disable IDE0044 // Add readonly modifier -#pragma warning disable CS0169 // Field is never used - private uint dwLength; - private uint dwMemoryLoad; - private ulong ullTotalPhys; -#pragma warning disable CS0649 // Field is never assigned to, and will always have its default value - private ulong ullAvailPhys; -#pragma warning restore CS0649 // Field is never assigned to, and will always have its default value - private ulong ullTotalPageFile; - private ulong ullAvailPageFile; - private ulong ullTotalVirtual; - private ulong ullAvailVirtual; - private ulong ullAvailExtendedVirtual; -#pragma warning restore CS0169 // Field is never used -#pragma warning restore IDE0044 // Add readonly modifier - - private MEMORYSTATUSEX() - { - dwLength = (uint)Marshal.SizeOf(typeof(MEMORYSTATUSEX)); - } - - internal static bool GetAvailablePhysicalMemory(out ulong availableMemory) - { - availableMemory = 0; - MEMORYSTATUSEX memStatus = new MEMORYSTATUSEX(); - if (GlobalMemoryStatusEx(memStatus)) - { - availableMemory = memStatus.ullAvailPhys; - return true; - } - - return false; - } - } } diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IGCInfoProvider.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IGCInfoProvider.cs new file mode 100644 index 000000000000..f3059d7a7a2e --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IGCInfoProvider.cs @@ -0,0 +1,34 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + /// + /// Abstraction for GC information + /// + internal interface IGCInfoProvider + { + /// + /// Gets the number of times garbage collection has occurred for gen 2 objects + /// + int GetGen2CollectionCount(); + +#if NETCOREAPP3_1_OR_GREATER + /// + /// Gets memory info from the garbage collector + /// + GCMemoryInfo GetGCMemoryInfo(); +#endif + + /// + /// Gets the memory usage ratio (0.0 to 1.0+) + /// + double GetMemoryUsageRatio(); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IHighResolutionClock.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IHighResolutionClock.cs new file mode 100644 index 000000000000..c2f84c5e4474 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IHighResolutionClock.cs @@ -0,0 +1,18 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal interface IHighResolutionClock + { + /// Gets the frequency of the timestamp (ticks per second). + double Frequency { get; } + + /// Gets the current timestamp in ticks. + long GetTimestamp(); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IMemoryPressureMonitor.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IMemoryPressureMonitor.cs new file mode 100644 index 000000000000..b509acf393dd --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IMemoryPressureMonitor.cs @@ -0,0 +1,46 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + /// + /// Monitors runtime memory pressure to protect against OOM scenarios. + /// + internal interface IMemoryPressureMonitor : IDisposable + { + /// + /// Gets a value indicating whether memory pressure is currently high + /// + bool IsHighMemoryPressure { get; } + + /// + /// Gets current memory usage as percentage (0-100) + /// + double MemoryUsagePercent { get; } + + /// + /// Gets Gen2 collections per second (indicator of pressure) + /// + double Gen2CollectionsPerSecond { get; } + + /// + /// Gets the high pressure memory threshold as a ratio (0-1) + /// + double HighPressureThreshold { get; } + + /// + /// Gets the maximum Gen2 collections per second threshold + /// + int MaxGen2PerSecond { get; } + + /// + /// Records memory pressure event + /// + void RecordMemoryPressureEvent(); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/ISamplerScheduler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/ISamplerScheduler.cs new file mode 100644 index 000000000000..acf30b5cea26 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/ISamplerScheduler.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + /// + /// Shared scheduler for sampler window rolls to reduce timer overhead + /// + internal interface ISamplerScheduler + { + /// + /// Schedules a callback to be invoked at regular intervals + /// + /// The callback to invoke + /// The interval at which to invoke the callback + /// A token that can be used to unschedule the callback + IDisposable Schedule(Action callback, TimeSpan interval); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureConfig.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureConfig.cs new file mode 100644 index 000000000000..6ea5a71c437b --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureConfig.cs @@ -0,0 +1,62 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Globalization; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal readonly struct MemoryPressureConfig + { + public static MemoryPressureConfig Default => new() + { + HighPressureThresholdRatio = 0.85, + MaxGen2PerSecond = 2, + MemoryExitMargin = 0.05, + Gen2ExitMargin = 1, + ConsecutiveHighToEnter = 1, + ConsecutiveLowToExit = 1, + RefreshInterval = TimeSpan.FromSeconds(1) + }; + + public double HighPressureThresholdRatio { get; init; } // 0.0–1.0 + + public int MaxGen2PerSecond { get; init; } + + public double MemoryExitMargin { get; init; } + + public int Gen2ExitMargin { get; init; } + + public int ConsecutiveHighToEnter { get; init; } + + public int ConsecutiveLowToExit { get; init; } + + public TimeSpan RefreshInterval { get; init; } + + public override string ToString() + { + var culture = CultureInfo.InvariantCulture; + var sb = StringBuilderCache.Acquire(); + + sb.Append("Threshold="); + sb.Append(HighPressureThresholdRatio.ToString("F2", culture)); + sb.Append(" ("); + sb.Append((HighPressureThresholdRatio * 100).ToString("F1", culture)); + sb.Append("%), MaxGen2="); + sb.Append(MaxGen2PerSecond.ToString(culture)); + sb.Append("/s, ExitMargin="); + sb.Append(MemoryExitMargin.ToString("F2", culture)); + sb.Append(", Gen2ExitMargin="); + sb.Append(Gen2ExitMargin.ToString(culture)); + sb.Append(", HighToEnter="); + sb.Append(ConsecutiveHighToEnter.ToString(culture)); + sb.Append(", LowToExit="); + sb.Append(ConsecutiveLowToExit.ToString(culture)); + + return StringBuilderCache.GetStringAndRelease(sb); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureMonitor.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureMonitor.cs new file mode 100644 index 000000000000..f169233ad54b --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryPressureMonitor.cs @@ -0,0 +1,290 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; +using System.Threading; +using Datadog.Trace.Logging; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + /// + /// Runtime memory pressure monitor for debugger probe protection. + /// Monitors system memory usage and GC Gen2 collection frequency with debounce + /// to avoid flapping between high and normal pressure states. + /// + internal sealed class MemoryPressureMonitor : IMemoryPressureMonitor + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private static readonly TimeSpan DefaultRefreshInterval = TimeSpan.FromSeconds(1); + + private static readonly TimerCallback TimerCallback = static state => + { + if (state is MemoryPressureMonitor @this) + { + @this.Refresh(); + } + }; + + // private readonly TimeSpan _refreshInterval; + private readonly IDisposable? _refreshTimer; + private readonly object _lock = new object(); + private readonly double _memoryExitMargin; + private readonly int _gen2ExitMargin; + private readonly int _consecutiveHighToEnter; + private readonly int _consecutiveLowToExit; + private readonly IGCInfoProvider _gcInfoProvider; + private readonly IHighResolutionClock _clock; + + // These fields are using inside a lock or with volatile read\write + private double _currentMemoryUsagePercent = 0; + private double _gen2CollectionsPerSecond = 0; + private volatile bool _isHighPressure = false; + private volatile bool _disabled = false; + private volatile bool _disposed = false; + + // Statistics tracking + private long _lastGen2Count = 0; + private long _lastRefreshTimestamp; + private long _pressureEventCount = 0; + private int _highStreak = 0; + private int _lowStreak = 0; + private bool _hasGen2Baseline = false; + + /// + /// Initializes a new instance of the class. + /// + /// Memory pressure config: + /// Memory usage ratio (0.0-1.0) to trigger high pressure. Default: 0.85 (85%) + /// Maximum Gen2 collections per second before triggering high pressure. Default: 2 + /// Margin below threshold before exiting high pressure. Default: 0.05 (5%) + /// Margin below maxGen2PerSecond before exiting high pressure. Default: 1 + /// Number of consecutive high cycles required to enter high pressure. Default: 1 + /// Number of consecutive low cycles required to exit high pressure. Default: 1 + /// + /// Optional custom scheduler. If null, uses Timer + /// Optional GC info provider. If null, uses + /// Optional clock. If null, uses + /// + /// Pressure events bypass consecutive cycle requirements and enter high pressure immediately. + /// + public MemoryPressureMonitor( + MemoryPressureConfig config, + ISamplerScheduler? scheduler = null, + IGCInfoProvider? gcInfoProvider = null, + IHighResolutionClock? clock = null) + { + HighPressureThreshold = config.HighPressureThresholdRatio; + MaxGen2PerSecond = config.MaxGen2PerSecond; + _memoryExitMargin = config.MemoryExitMargin; + _gen2ExitMargin = config.Gen2ExitMargin; + _consecutiveHighToEnter = Math.Max(1, config.ConsecutiveHighToEnter); + _consecutiveLowToExit = Math.Max(1, config.ConsecutiveLowToExit); + _gcInfoProvider = gcInfoProvider ?? new SystemGCInfoProvider(); + _clock = clock ?? new SystemClock(); + _lastRefreshTimestamp = _clock.GetTimestamp(); + _lastGen2Count = 0; // Defer baseline to first Refresh() call + _hasGen2Baseline = false; + + _refreshTimer = scheduler != null + ? scheduler.Schedule(Refresh, config.RefreshInterval) + : new Timer(TimerCallback, this, config.RefreshInterval, config.RefreshInterval); + + Refresh(); + } + + public double Gen2CollectionsPerSecond => Volatile.Read(ref _gen2CollectionsPerSecond); + + public double MemoryUsagePercent => Volatile.Read(ref _currentMemoryUsagePercent); + + public bool IsHighMemoryPressure => _isHighPressure; + + public double HighPressureThreshold { get; } + + public int MaxGen2PerSecond { get; } + + public void RecordMemoryPressureEvent() + { + Interlocked.Increment(ref _pressureEventCount); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _refreshTimer?.Dispose(); + } + + private void Refresh() + { + if (_disposed || _disabled) + { + return; + } + + try + { + var now = _clock.GetTimestamp(); + + // 1) Read + double memRatio = 0; + var memAvailable = true; + try + { + memRatio = _gcInfoProvider.GetMemoryUsageRatio(); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to get memory usage ratio"); + memAvailable = false; + } + + var gen2Count = 0; + var gcAvailable = true; + try + { + gen2Count = _gcInfoProvider.GetGen2CollectionCount(); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to get Gen2 collection count"); + gcAvailable = false; + } + + var pressureEvents = Interlocked.Exchange(ref _pressureEventCount, 0); + + // 2) If nothing is available, disable the monitor + if (!memAvailable && !gcAvailable) + { + lock (_lock) + { + _isHighPressure = false; + Volatile.Write(ref _currentMemoryUsagePercent, 0); + Volatile.Write(ref _gen2CollectionsPerSecond, 0); + _disabled = true; + _refreshTimer?.Dispose(); + } + + Log.Debug("MemoryPressureMonitor disabled: no memory or GC info available on this runtime/platform."); + return; + } + + // 3) Establish Gen2 baseline if needed + if (!_hasGen2Baseline && gcAvailable) + { + lock (_lock) + { + _lastRefreshTimestamp = now; + _lastGen2Count = gen2Count; + _hasGen2Baseline = true; + } + } + + // 4) Compute rates and update state + var exitMemThreshold = Math.Max(0, HighPressureThreshold - _memoryExitMargin); + var exitGen2Threshold = Math.Max(0, MaxGen2PerSecond - _gen2ExitMargin); + + bool transition; + bool newHigh; + double logUsagePercent; + double logGen2PerSecond; + + lock (_lock) + { + var elapsedSeconds = (now - _lastRefreshTimestamp) / _clock.Frequency; + + double gen2PerSecond = 0; + if (_hasGen2Baseline && gcAvailable) + { + var delta = gen2Count - _lastGen2Count; + if (delta < 0) + { + delta = 0; + } + + _lastGen2Count = gen2Count; + gen2PerSecond = elapsedSeconds > 0 ? delta / elapsedSeconds : 0; + } + + // Threshold checks + var hasEvent = pressureEvents > 0; + var aboveEnterMem = memAvailable && (memRatio > HighPressureThreshold); + var aboveExitMem = memAvailable && (memRatio > exitMemThreshold); + var aboveEnterGc = _hasGen2Baseline && gcAvailable && (gen2PerSecond > MaxGen2PerSecond); + var aboveExitGc = _hasGen2Baseline && gcAvailable && (gen2PerSecond > exitGen2Threshold); + + var meetsHighNow = _isHighPressure + ? (aboveExitMem || aboveExitGc || hasEvent) + : (aboveEnterMem || aboveEnterGc || hasEvent); + + bool nextHigh = ComputeNextHigh(meetsHighNow, hasEvent); + + Volatile.Write(ref _currentMemoryUsagePercent, memRatio * 100); + Volatile.Write(ref _gen2CollectionsPerSecond, gen2PerSecond); + + var prev = _isHighPressure; + _isHighPressure = nextHigh; + _lastRefreshTimestamp = now; + + transition = nextHigh != prev; + newHigh = nextHigh; + logUsagePercent = _currentMemoryUsagePercent; + logGen2PerSecond = _gen2CollectionsPerSecond; + } + + if (transition) + { + Log.Debug( + "Memory pressure {State}: Usage={Usage:F1}%, Gen2/sec={Gen2:F2}", + property0: newHigh ? "ENTER" : "EXIT", + property1: logUsagePercent, + property2: logGen2PerSecond); + } + } + catch (Exception ex) + { + Log.Error(ex, "Error refreshing memory pressure"); + } + + bool ComputeNextHigh(bool meetsHighNow, bool hasEvent) + { + if (hasEvent) + { + _highStreak = _consecutiveHighToEnter; + _lowStreak = 0; + return true; + } + + if (meetsHighNow) + { + if (_highStreak < _consecutiveHighToEnter) + { + _highStreak++; + } + + _lowStreak = 0; + } + else + { + if (_lowStreak < _consecutiveLowToExit) + { + _lowStreak++; + } + + _highStreak = 0; + } + + return _isHighPressure + ? _lowStreak < _consecutiveLowToExit // stay high until enough low cycles + : _highStreak >= _consecutiveHighToEnter; // enter after enough high cycles + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemClock.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemClock.cs new file mode 100644 index 000000000000..87efc97dea9a --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemClock.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Diagnostics; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + /// + /// Default implementation that uses system Stopwatch and DateTime + /// + internal sealed class SystemClock : IHighResolutionClock + { + public double Frequency => Stopwatch.Frequency; + + public long GetTimestamp() + { + return Stopwatch.GetTimestamp(); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemGCInfoProvider.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemGCInfoProvider.cs new file mode 100644 index 000000000000..76d75f561cd3 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/SystemGCInfoProvider.cs @@ -0,0 +1,85 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +#if NETFRAMEWORK +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.RateLimiting; + +#endif +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + /// + /// Default implementation that uses real GC and system memory APIs + /// + internal sealed class SystemGCInfoProvider : IGCInfoProvider + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + + public int GetGen2CollectionCount() + { + return GC.CollectionCount(2); + } + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() + { + return GC.GetGCMemoryInfo(); + } +#endif + + public double GetMemoryUsageRatio() + { + double memoryUsageRatio = 0; +#if NETCOREAPP3_1_OR_GREATER + try + { + var gcInfo = GetGCMemoryInfo(); + if (gcInfo.HighMemoryLoadThresholdBytes > 0) + { + memoryUsageRatio = (double)gcInfo.MemoryLoadBytes / gcInfo.HighMemoryLoadThresholdBytes; + } + else + { + Log.Debug( + "GC.GetGCMemoryInfo returned non-positive HighMemoryLoadThresholdBytes. Treating memory usage ratio as unsupported on this runtime."); + } + } + catch (Exception ex) + { + Log.Error(ex, "Failed to get GC memory info"); + } +#elif NETFRAMEWORK + try + { + if (WindowsMemoryInfo.TryGetMemoryLoadRatio(out var ratio)) + { + // Windows GlobalMemoryStatusEx returns dwMemoryLoad (0-100) which is clamped + // to [0,1] range. This differs from .NET Core GC path which can exceed 1.0. + memoryUsageRatio = ratio; + } + else + { + Log.Debug( + "GlobalMemoryStatusEx did not return memory information. Treating memory usage ratio as unsupported on this platform."); + } + } + catch (Exception ex) + { + Log.Error(ex, "Failed to get system memory status"); + } +#else + Log.Debug( + "Memory usage ratio is not supported on this runtime target. Not NETCOREAPP3_1_OR_GREATER or NETFRAMEWORK; defaulting to 0."); + memoryUsageRatio = 0; +#endif + return memoryUsageRatio; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/WindowsMemoryInfo.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/WindowsMemoryInfo.cs new file mode 100644 index 000000000000..0bfd7ac9c3d7 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/WindowsMemoryInfo.cs @@ -0,0 +1,72 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal static class WindowsMemoryInfo + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(WindowsMemoryInfo)); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GlobalMemoryStatusEx(ref MEMORYSTATUSEX lpBuffer); + + internal static bool TryGetMemoryLoadRatio(out double ratio) + { + var status = new MEMORYSTATUSEX { dwLength = (uint)Marshal.SizeOf() }; + if (GlobalMemoryStatusEx(ref status)) + { + ratio = Math.Min(1.0, status.dwMemoryLoad / 100.0); + return true; + } + + var error = Marshal.GetLastWin32Error(); + Log.Debug( + "GlobalMemoryStatusEx failed when getting memory load ratio. ErrorCode={ErrorCode}", + property: error); + ratio = 0; + return false; + } + + internal static bool TryGetAvailablePhysicalMemory(out ulong bytes) + { + var status = new MEMORYSTATUSEX { dwLength = (uint)Marshal.SizeOf() }; + if (GlobalMemoryStatusEx(ref status)) + { + bytes = status.ullAvailPhys; + return true; + } + + var error = Marshal.GetLastWin32Error(); + Log.Debug( + "GlobalMemoryStatusEx failed when getting available physical memory. ErrorCode={ErrorCode}", + property: error); + bytes = 0; + return false; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] + private struct MEMORYSTATUSEX + { +#pragma warning disable SA1307 + public uint dwLength; + public uint dwMemoryLoad; + public ulong ullTotalPhys; + public ulong ullAvailPhys; + public ulong ullTotalPageFile; + public ulong ullAvailPageFile; + public ulong ullTotalVirtual; + public ulong ullAvailVirtual; + public ulong ullAvailExtendedVirtual; +#pragma warning restore SA1307 + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeClock.cs b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeClock.cs new file mode 100644 index 000000000000..a385bbbd460d --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeClock.cs @@ -0,0 +1,42 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using System.Diagnostics; +using Datadog.Trace.Debugger.RateLimiting; + +namespace Datadog.Trace.Tests.Debugger.RateLimiting +{ + /// + /// Fake clock for deterministic time testing without real time dependencies + /// + internal sealed class FakeClock(double? frequency = null) : IHighResolutionClock + { + private readonly Queue _timestamps = new(); + private long _currentTimestamp = 0; + + public double Frequency { get; } = frequency ?? Stopwatch.Frequency; + + public FakeClock WithTicksAtSeconds(params double[] seconds) + { + foreach (var second in seconds) + { + _timestamps.Enqueue((long)(second * this.Frequency)); + } + + return this; + } + + public long GetTimestamp() + { + if (_timestamps.Count > 0) + { + _currentTimestamp = _timestamps.Dequeue(); + } + + return _currentTimestamp; + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeGCInfoProvider.cs b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeGCInfoProvider.cs new file mode 100644 index 000000000000..39279b623244 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/FakeGCInfoProvider.cs @@ -0,0 +1,83 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using Datadog.Trace.Debugger.RateLimiting; + +namespace Datadog.Trace.Tests.Debugger.RateLimiting +{ + /// + /// Fake GC info provider for deterministic testing without real GC interactions + /// + internal class FakeGCInfoProvider : IGCInfoProvider + { + private readonly Queue _gen2Counts = new(); + private readonly Queue _memoryRatios = new(); + private double _currentMemoryRatio = 0.5; + private int _currentGen2Count = 0; + + public FakeGCInfoProvider WithGen2Counts(params int[] counts) + { + foreach (var count in counts) + { + _gen2Counts.Enqueue(count); + } + + return this; + } + + public FakeGCInfoProvider WithMemoryRatios(params double[] ratios) + { + foreach (var ratio in ratios) + { + _memoryRatios.Enqueue(ratio); + } + + return this; + } + + public FakeGCInfoProvider WithConstantGen2Count(int count) + { + _currentGen2Count = count; + return this; + } + + public FakeGCInfoProvider WithConstantMemoryRatio(double ratio) + { + _currentMemoryRatio = ratio; + return this; + } + + public int GetGen2CollectionCount() + { + if (_gen2Counts.Count > 0) + { + _currentGen2Count = _gen2Counts.Dequeue(); + } + + return _currentGen2Count; + } + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() + { + // Return a default GCMemoryInfo - not used in current tests + // but needed for interface completeness + return new GCMemoryInfo(); + } +#endif + + public double GetMemoryUsageRatio() + { + if (_memoryRatios.Count > 0) + { + _currentMemoryRatio = _memoryRatios.Dequeue(); + } + + return _currentMemoryRatio; + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/MemoryPressureMonitorTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/MemoryPressureMonitorTests.cs new file mode 100644 index 000000000000..123261234632 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/MemoryPressureMonitorTests.cs @@ -0,0 +1,906 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.RateLimiting; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger.RateLimiting +{ +#if NETFRAMEWORK || NETCOREAPP3_1_OR_GREATER + public class MemoryPressureMonitorTests + { + [Fact] + public void Gen2CollectionsPerSecond_TracksGC() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithGen2Counts(0, 3) + .WithConstantMemoryRatio(0.1); + var clock = new FakeClock().WithTicksAtSeconds(0, 0, 1, 1); + + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.95, + MaxGen2PerSecond = 100 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + scheduler.TriggerRefresh(); + + var rate = monitor.Gen2CollectionsPerSecond; + + // Should be 3 collections per second + rate.Should().BeApproximately(3.0, 0.01); + } + + [Fact] + public async Task ConcurrentReadsDuringRefresh_NoDeadlock() + { + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.80, + MaxGen2PerSecond = 2 + }; + + using var monitor = new MemoryPressureMonitor(config); + + var readCount = 0; + var exceptions = 0; + var duration = TimeSpan.FromSeconds(2); + var sw = System.Diagnostics.Stopwatch.StartNew(); + + // Read while refresh is happening every timer interval + var tasks = Enumerable.Range(0, 8) + .Select(i => Task.Run(() => + { + try + { + while (sw.Elapsed < duration) + { + _ = monitor.IsHighMemoryPressure; + _ = monitor.MemoryUsagePercent; + _ = monitor.Gen2CollectionsPerSecond; + Interlocked.Increment(ref readCount); + } + } + catch + { + Interlocked.Increment(ref exceptions); + } + })) + .ToArray(); + + await Task.WhenAll(tasks); + + exceptions.Should().Be(0, "No exceptions should occur during concurrent reads"); + readCount.Should().BeGreaterThan(1000, "Should achieve reasonable read throughput without lock contention"); + } + + [Fact] + public void Constructor_AcceptsExtremeParameterValues() + { + using var scheduler = new TestScheduler(); + + // Test negative values + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = -0.1, + MaxGen2PerSecond = -5 + }; + + using var monitor1 = new MemoryPressureMonitor(config, scheduler); + + // Test values > 1 + config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 2, + }; + using var monitor2 = new MemoryPressureMonitor(config, scheduler: scheduler); + + // Test zero values + config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0, + MaxGen2PerSecond = 0 + }; + using var monitor3 = new MemoryPressureMonitor(config, scheduler: scheduler); + + // All should initialize without throwing + monitor1.Should().NotBeNull(); + monitor2.Should().NotBeNull(); + monitor3.Should().NotBeNull(); + } + + [Fact] + public void WindowsMemoryInfo_TryGetMemoryLoadRatio_ReturnsValidValue() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return; + } + + var result = WindowsMemoryInfo.TryGetMemoryLoadRatio(out var ratio); + + result.Should().BeTrue(); + ratio.Should().BeGreaterThan(0); + ratio.Should().BeLessOrEqualTo(1); + } + + [Fact] + public void WindowsMemoryInfo_TryGetAvailablePhysicalMemory_ReturnsValidValue() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return; + } + + var result = WindowsMemoryInfo.TryGetAvailablePhysicalMemory(out var bytes); + + result.Should().BeTrue(); + bytes.Should().BeGreaterThan(0); + } + + [Fact] + public void Dispose_PreventsFurtherOperations() + { + using var scheduler = new TestScheduler(); + var monitor = new MemoryPressureMonitor(MemoryPressureConfig.Default, scheduler: scheduler); + + monitor.Dispose(); + + // Further operations should not throw but may not update + scheduler.TriggerRefresh(); + + // Access properties - should not throw + var usage = monitor.MemoryUsagePercent; + var gen2Rate = monitor.Gen2CollectionsPerSecond; + var isHigh = monitor.IsHighMemoryPressure; + + // Values should be valid (last known values or defaults) + usage.Should().BeGreaterOrEqualTo(0); + gen2Rate.Should().BeGreaterOrEqualTo(0); + } + + [Fact] + public void Dispose_StopsScheduledRefresh() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithConstantMemoryRatio(0.10) // below threshold + .WithConstantGen2Count(0); // no GC pressure + var clock = new FakeClock().WithTicksAtSeconds(0, 1); + + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.99, + MaxGen2PerSecond = 1000 + }; + + var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + monitor.IsHighMemoryPressure.Should().BeFalse(); + + // Queue an event that would flip to high on next refresh + monitor.RecordMemoryPressureEvent(); + + // Dispose should remove the scheduled callback; triggering should not process the event + monitor.Dispose(); + scheduler.TriggerRefresh(); + + monitor.IsHighMemoryPressure.Should().BeFalse(); + } + + [Fact] + public void ConsecutiveLowToExit_DelaysExit() + { + using var scheduler = new TestScheduler(); + var config = new MemoryPressureConfig + { + HighPressureThresholdRatio = 2.0, + MaxGen2PerSecond = 1000, + MemoryExitMargin = 0.0, + Gen2ExitMargin = 0, + ConsecutiveHighToEnter = 1, + ConsecutiveLowToExit = 2 + }; + + using var monitor = new MemoryPressureMonitor(config, scheduler); + + // Enter high via pressure event (bypasses other checks) + monitor.RecordMemoryPressureEvent(); + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // No new events; low cycle #1: should remain high + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Low cycle #2: should EXIT + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + } + + [Fact] + public void PressureEvent_BypassesConsecutiveHighRequirement() + { + using var scheduler = new TestScheduler(); + var config = new MemoryPressureConfig + { + HighPressureThresholdRatio = 0.99, + MaxGen2PerSecond = 1000, + ConsecutiveHighToEnter = 3, + ConsecutiveLowToExit = 1 + }; + + using var monitor = new MemoryPressureMonitor(config, scheduler); + + monitor.RecordMemoryPressureEvent(); + scheduler.TriggerRefresh(); + + // Should enter immediately due to event bypass + monitor.IsHighMemoryPressure.Should().BeTrue(); + } + + [Fact] + public void PressureEvents_ResetToZero_AfterProcessing() + { + using var scheduler = new TestScheduler(); + + var config = new MemoryPressureConfig + { + HighPressureThresholdRatio = 2.0, + MaxGen2PerSecond = 1000, + MemoryExitMargin = 0.0, + Gen2ExitMargin = 0 + }; + + using var monitor = new MemoryPressureMonitor(config, scheduler); + + // Record events + monitor.RecordMemoryPressureEvent(); + monitor.RecordMemoryPressureEvent(); + + // First refresh processes them + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Record one more event + monitor.RecordMemoryPressureEvent(); + + // Second refresh processes the new event + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Third refresh with no new events should exit high pressure + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + } + + [Fact] + public void ThresholdBoundary_JustAboveMemoryThreshold_Enters() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithConstantMemoryRatio(0.8006); // Just above threshold (0.80) + var clock = new FakeClock().WithTicksAtSeconds(0, 1); + + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.80, + MaxGen2PerSecond = 1000 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + scheduler.TriggerRefresh(); + + // Should enter because > threshold + monitor.IsHighMemoryPressure.Should().BeTrue(); + } + + [Fact] + public void Monitor_Disables_WhenNoSignalsAvailable() + { + using var scheduler = new TestScheduler(); + var clock = new FakeClock().WithTicksAtSeconds(0, 1, 2); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.80, + MaxGen2PerSecond = 2 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: new NoSignalsGCInfoProvider(), + clock: clock); + + // Constructor performs an initial refresh and disables immediately -> no subscription + scheduler.SubscriptionCount.Should().Be(0); + + // First refresh should detect no signals and disable the monitor + scheduler.TriggerRefresh(); + + monitor.IsHighMemoryPressure.Should().BeFalse(); + monitor.MemoryUsagePercent.Should().Be(0); + monitor.Gen2CollectionsPerSecond.Should().Be(0); + + // Monitor should have unsubscribed + scheduler.SubscriptionCount.Should().Be(0); + + // Further refresh calls should have no effect + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + monitor.MemoryUsagePercent.Should().Be(0); + monitor.Gen2CollectionsPerSecond.Should().Be(0); + } + + [Fact] + public void MemoryOnly_AppliesThresholds_WhenGcUnavailable() + { + using var scheduler = new TestScheduler(); + var clock = new FakeClock().WithTicksAtSeconds(0, 1, 2, 3); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.80, + MaxGen2PerSecond = 1000 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: new MemoryOnlyGCInfoProvider(0.50, 0.85, 0.74), // Add a below-threshold sample for ctor refresh, then enter, then exit + clock: clock); + + // First refresh: memory 0.85 > 0.80 => enter high + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + monitor.Gen2CollectionsPerSecond.Should().Be(0); + + // Second refresh: memory 0.74 < 0.80 - 0.05 (exit threshold 0.75) => exit + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + monitor.Gen2CollectionsPerSecond.Should().Be(0); + } + + [Fact] + public void GcOnly_AppliesThresholds_WhenMemoryUnavailable() + { + using var scheduler = new TestScheduler(); + var clock = new FakeClock().WithTicksAtSeconds(0, 0, 1, 1); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.99, + MaxGen2PerSecond = 2 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: new GCOnlyInfoProvider([0, 3]), + clock: clock); + + // First scheduled refresh: ctor has already established baseline; rate is 3/sec > 2 => enter high + scheduler.TriggerRefresh(); + monitor.Gen2CollectionsPerSecond.Should().BeApproximately(3.0, 0.01); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Second refresh: no additional collections => rate ~0 + scheduler.TriggerRefresh(); + monitor.Gen2CollectionsPerSecond.Should().BeLessOrEqualTo(0.001); + } + + [Fact] + public void ThresholdBoundary_JustBelowExitThreshold_RemainsHigh() + { + using var scheduler = new TestScheduler(); + + // Add an initial below-threshold sample to offset constructor's initial refresh + var gc = new FakeGCInfoProvider() + .WithMemoryRatios(0.70, 0.90, 0.8001); // Enter high on 0.90, then drop to just above exit (needs 0.05 to exit) + var clock = new FakeClock().WithTicksAtSeconds(0, 1, 2, 3); + + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.85, + MaxGen2PerSecond = 1000, + MemoryExitMargin = 0.05 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + // Enter high pressure + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Drop to just above exit threshold (0.8001 > 0.80) + scheduler.TriggerRefresh(); + + // Should remain high + monitor.IsHighMemoryPressure.Should().BeTrue(); + } + + [Fact] + public void ThresholdBoundary_JustAboveGen2Threshold_Enters() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithGen2Counts(0, 3) // 3 Gen2/sec > threshold + .WithConstantMemoryRatio(0.1); + var clock = new FakeClock().WithTicksAtSeconds(0, 0, 1, 1); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.99, + MaxGen2PerSecond = 2, + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + scheduler.TriggerRefresh(); + + // Should enter because > threshold + monitor.IsHighMemoryPressure.Should().BeTrue(); + } + + [Fact] + public void Gen2ExitMargin_Hysteresis_ExitOnlyWhenBelowMargin() + { + using var scheduler = new TestScheduler(); + + // Counts: baseline (ctor refresh) at 0, then +3, then +2, then +1 + // Rates: 3/sec (enter), 2/sec (>1 remain high), 1/sec (==1 exit) + var gcOnly = new GCOnlyInfoProvider([0, 3, 5, 6]); + var clock = new FakeClock().WithTicksAtSeconds(0, 0, 1, 2, 3); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 1.0, + MaxGen2PerSecond = 2, + Gen2ExitMargin = 1 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gcOnly, + clock: clock); + + // First scheduled refresh: 3/sec -> ENTER + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Second: 2/sec (>1) -> remain HIGH + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Third: 1/sec (==1) -> EXIT + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + } + + [Fact] + public void ConsecutiveHighToEnter_WithMetrics_RequiresMultipleCycles() + { + using var scheduler = new TestScheduler(); + var memOnly = new MemoryOnlyGCInfoProvider(0.50, 0.86, 0.86); // below for ctor; then above twice + var clock = new FakeClock().WithTicksAtSeconds(0, 1, 2, 3); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.85, + MaxGen2PerSecond = 1000, + ConsecutiveHighToEnter = 2 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: memOnly, + clock: clock); + + // First above-threshold cycle -> not enough to enter + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + + // Second consecutive above-threshold -> ENTER + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + } + + [Fact] + public void EstablishesGen2Baseline_OnFirstSuccessfulRefresh() + { + // This test exercises the generic IGCInfoProvider contract, not the current SystemGCInfoProvider. + // It simulates a transient GC failure during the ctor's initial Refresh: + // - First call to GetGen2CollectionCount() throws, so no Gen2 baseline is established yet. + // - A later refresh, once the provider starts succeeding, establishes the baseline (rate still 0). + // - Subsequent refreshes then compute non-zero rates from that baseline and can enter high pressure. + // With the current SystemGCInfoProvider implementation GC.CollectionCount(2) is not expected to throw, + // but we validate that MemoryPressureMonitor behaves correctly if a custom or future provider does. + + using var scheduler = new TestScheduler(); + var gc = new TransientGcThrowProvider( + initialMemoryRatio: 0.1, + countsAfterRecovery: new[] { 0, 3, 6 }); // after recovery: baseline 0, then 3, then 6 + var clock = new FakeClock().WithTicksAtSeconds(0, 0, 1, 2, 3); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 1.0, + MaxGen2PerSecond = 2, + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + // First scheduled refresh: GC threw in ctor; this establishes baseline (rate still 0) + scheduler.TriggerRefresh(); + monitor.Gen2CollectionsPerSecond.Should().Be(0); + monitor.IsHighMemoryPressure.Should().BeFalse(); + + // Second scheduled refresh: depending on timestamp alignment, rate may still be ~0 + scheduler.TriggerRefresh(); + monitor.Gen2CollectionsPerSecond.Should().BeGreaterOrEqualTo(0); + + // Third scheduled refresh: delta of 3 over 1 second => ~3/sec and ENTER high + scheduler.TriggerRefresh(); + monitor.Gen2CollectionsPerSecond.Should().BeApproximately(3.0, 0.05); + monitor.IsHighMemoryPressure.Should().BeTrue(); + } + + [Fact] + public void MemoryUsagePercent_ScalesRatioToPercent() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithConstantMemoryRatio(0.80) + .WithConstantGen2Count(0); + var clock = new FakeClock().WithTicksAtSeconds(0, 1); + + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 2.0, + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + scheduler.TriggerRefresh(); + + monitor.MemoryUsagePercent.Should().BeApproximately(80.0, 0.1); + } + + [Fact] + public void CycleWithoutTime_Gen2RateDoesNotInflate() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithGen2Counts(0, 2, 4); + var clock = new FakeClock() + .WithTicksAtSeconds(1.0, 1.0, 1.0); // No time progress! + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.99, + MaxGen2PerSecond = 1000 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + // First refresh: sets baseline + scheduler.TriggerRefresh(); + + // Second refresh: no time elapsed, should compute rate as 0 + scheduler.TriggerRefresh(); + + var rate = monitor.Gen2CollectionsPerSecond; + + // Should be 0 or very small, not infinity or large number + rate.Should().BeLessOrEqualTo(0.001); + } + + [Fact] + public void SchedulerDispose_ThenTriggerRefresh_DoesNotThrow() + { + var scheduler = new TestScheduler(); + using var monitor = new MemoryPressureMonitor(MemoryPressureConfig.Default, scheduler: scheduler); + + scheduler.Dispose(); + + // Should not throw even though scheduler is disposed + Action act = () => scheduler.TriggerRefresh(); + act.Should().NotThrow(); + + // Monitor should still be accessible + monitor.MemoryUsagePercent.Should().BeGreaterOrEqualTo(0); + } + + [Fact] + public void NegativeGen2Delta_TreatedAsZero() + { + using var scheduler = new TestScheduler(); + var gc = new FakeGCInfoProvider() + .WithGen2Counts(100, 50); // Count goes backward (shouldn't happen in reality) + var clock = new FakeClock().WithTicksAtSeconds(0, 0, 1, 1); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.99, + MaxGen2PerSecond = 1000 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + scheduler.TriggerRefresh(); + + var rate = monitor.Gen2CollectionsPerSecond; + + // Should be 0, not negative + rate.Should().Be(0); + } + + [Fact] + public void ExitMargin_WithPositiveMargin_ExitsOnlyBelowMargin() + { + using var scheduler = new TestScheduler(); + + // Add an initial below-threshold sample to offset constructor's initial refresh + var gc = new FakeGCInfoProvider() + .WithMemoryRatios(0.50, 0.85, 0.85, 0.81, 0.76, 0.74) // Enter high, stay, drop gradually + .WithConstantGen2Count(0); + + var clock = new FakeClock().WithTicksAtSeconds(0, 1, 2, 3, 4, 5, 6); + + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.80, + MaxGen2PerSecond = 1000, + MemoryExitMargin = 0.05 + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + // Enter (0.85 > 0.80) + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Stay high (0.85 > 0.75) + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Still high (0.81 > 0.75) + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Still high (0.76 > 0.75) + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeTrue(); + + // Exit (0.74 <= 0.75) + scheduler.TriggerRefresh(); + monitor.IsHighMemoryPressure.Should().BeFalse(); + } + + [Fact] + public void ProviderException_DoesNotCrashMonitor() + { + using var scheduler = new TestScheduler(); + var gc = new ThrowingGCInfoProvider(); + var clock = new FakeClock().WithTicksAtSeconds(0, 1, 2); + var config = MemoryPressureConfig.Default with + { + HighPressureThresholdRatio = 0.80, + MaxGen2PerSecond = 10, + }; + + using var monitor = new MemoryPressureMonitor( + config, + scheduler: scheduler, + gcInfoProvider: gc, + clock: clock); + + // Constructor performs an initial refresh that disables due to no signals -> no subscription + scheduler.SubscriptionCount.Should().Be(0); + + // Refresh should not throw even if provider throws + Action act = () => scheduler.TriggerRefresh(); + act.Should().NotThrow(); + + // Properties should still be accessible (with default/previous values) + monitor.MemoryUsagePercent.Should().BeGreaterOrEqualTo(0); + monitor.Gen2CollectionsPerSecond.Should().BeGreaterOrEqualTo(0); + + // Should not enter high pressure due to provider failures + monitor.IsHighMemoryPressure.Should().BeFalse(); + + // Monitor should disable itself (unsubscribe) as no signals are available + scheduler.SubscriptionCount.Should().Be(0); + } + + /// + /// GC provider that throws for both memory and GC info, simulating unsupported platform. + /// + private class NoSignalsGCInfoProvider : IGCInfoProvider + { + public int GetGen2CollectionCount() => throw new InvalidOperationException("No GC info available"); + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() => new GCMemoryInfo(); +#endif + + public double GetMemoryUsageRatio() => throw new InvalidOperationException("No memory info available"); + } + + /// + /// GC provider that supplies memory ratios only and throws for GC counts. + /// + private class MemoryOnlyGCInfoProvider : IGCInfoProvider + { + private readonly System.Collections.Generic.Queue _ratios = new(); + + public MemoryOnlyGCInfoProvider(params double[] ratios) + { + foreach (var r in ratios) + { + _ratios.Enqueue(r); + } + } + + public int GetGen2CollectionCount() => throw new InvalidOperationException("GC count unavailable"); + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() => new GCMemoryInfo(); +#endif + + public double GetMemoryUsageRatio() + { + if (_ratios.Count == 0) + { + return 0; + } + + return _ratios.Dequeue(); + } + } + + /// + /// GC provider that supplies Gen2 counts only and throws for memory ratio. + /// + private class GCOnlyInfoProvider : IGCInfoProvider + { + private readonly System.Collections.Generic.Queue _counts = new(); + + public GCOnlyInfoProvider(int[] counts) + { + foreach (var c in counts) + { + _counts.Enqueue(c); + } + } + + public int GetGen2CollectionCount() + { + if (_counts.Count == 0) + { + return 0; + } + + return _counts.Dequeue(); + } + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() => new GCMemoryInfo(); +#endif + + public double GetMemoryUsageRatio() => throw new InvalidOperationException("Memory ratio unavailable"); + } + + /// + /// Helper provider that throws exceptions to test error handling + /// + private class ThrowingGCInfoProvider : IGCInfoProvider + { + public int GetGen2CollectionCount() + { + throw new InvalidOperationException("Test exception from GC provider"); + } + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() + { + throw new InvalidOperationException("Test exception from GC provider"); + } +#endif + + public double GetMemoryUsageRatio() + { + throw new InvalidOperationException("Test exception from memory provider"); + } + } + + /// + /// GC provider that throws for GetGen2CollectionCount() on first invocation only, + /// then returns provided counts. Memory ratio always available and low. + /// + private class TransientGcThrowProvider : IGCInfoProvider + { + private readonly System.Collections.Generic.Queue _counts = new(); + private readonly double _memoryRatio; + private bool _hasThrown = false; + + public TransientGcThrowProvider(double initialMemoryRatio, int[] countsAfterRecovery) + { + _memoryRatio = initialMemoryRatio; + foreach (var c in countsAfterRecovery) + { + _counts.Enqueue(c); + } + } + + public int GetGen2CollectionCount() + { + if (!_hasThrown) + { + _hasThrown = true; + throw new InvalidOperationException("Transient GC failure"); + } + + if (_counts.Count == 0) + { + return 0; + } + + return _counts.Dequeue(); + } + +#if NETCOREAPP3_1_OR_GREATER + public GCMemoryInfo GetGCMemoryInfo() => new GCMemoryInfo(); +#endif + + public double GetMemoryUsageRatio() => _memoryRatio; + } + } +#endif +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/TestScheduler.cs b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/TestScheduler.cs new file mode 100644 index 000000000000..9a28e078e4b4 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/RateLimiting/TestScheduler.cs @@ -0,0 +1,71 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Linq; +using Datadog.Trace.Debugger.RateLimiting; + +namespace Datadog.Trace.Tests.Debugger.RateLimiting +{ + /// + /// Test scheduler that allows manual control over when callbacks are invoked. + /// + internal class TestScheduler : ISamplerScheduler, IDisposable + { + private readonly System.Collections.Generic.List<(Action Callback, IDisposable Token)> _scheduled = []; + private bool _disposed; + + public int SubscriptionCount => _scheduled.Count; + + public IDisposable Schedule(Action callback, TimeSpan interval) + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(TestScheduler)); + } + + var token = new TestScheduleToken(this); + _scheduled.Add((callback, token)); + return token; + } + + public void TriggerRefresh() + { + if (_disposed) + { + return; + } + + // Create a copy to avoid issues if callbacks modify the list + var callbacks = _scheduled.Select(x => x.Callback).ToArray(); + foreach (var callback in callbacks) + { + callback(); + } + } + + public void Dispose() + { + _disposed = true; + _scheduled.Clear(); + } + + private class TestScheduleToken : IDisposable + { + private readonly TestScheduler _scheduler; + + public TestScheduleToken(TestScheduler scheduler) + { + _scheduler = scheduler; + } + + public void Dispose() + { + // Remove from scheduler's list + _scheduler._scheduled.RemoveAll(x => Equals(x.Token, this)); + } + } + } +}