Memory Randomization (#1587)

* add initial implementation of experimental mode that allocates random-size array between iterations and calls global setup after it

* don't remove outliers if Memory Randomization is enabled

* introduce an attribute that allows to enable MemoryRandomization per class

* rename ForceAllocations to ForceGcCleanups to make it clear what it does

* add a call to [GlobalCleanup] before [GlobalSetu]

allocate sth on LOH too

* add stack memory randomization

* keep the stack memory alive for the iteration period

Author: adamsitnik

samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs

@@ -0,0 +1,28 @@
+using BenchmarkDotNet.Attributes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BenchmarkDotNet.Samples
+{
+    public class IntroMemoryRandomization
+    {
+        [Params(512 * 4)]
+        public int Size;
+
+        private int[] _array;
+        private int[] _destination;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            _array = new int[Size];
+            _destination = new int[Size];
+        }
+
+        [Benchmark]
+        public void Array() => System.Array.Copy(_array, _destination, Size);
+    }
+}

src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs

@@ -0,0 +1,17 @@
+using BenchmarkDotNet.Jobs;
+using Perfolizer.Mathematics.OutlierDetection;
+
+namespace BenchmarkDotNet.Attributes
+{
+    /// <summary>
+    /// specifies whether Engine should allocate some random-sized memory between iterations
+    /// <remarks>it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration</remarks>
+    /// </summary>
+    public class MemoryRandomizationAttribute : JobMutatorConfigBaseAttribute
+    {
+        public MemoryRandomizationAttribute(bool enable = true, OutlierMode outlierMode = OutlierMode.DontRemove)
+            : base(Job.Default.WithMemoryRandomization(enable).WithOutlierMode(outlierMode))
+        {
+        }
+    }
+}

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -168,6 +168,9 @@ public class CommandLineOptions
         [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")]
         public IEnumerable<string> EnvironmentVariables { get; set; }
 
+        [Option("memoryRandomization", Required = false, HelpText = "Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.")]
+        public bool MemoryRandomization { get; set; }
+
         [Option("wasmEngine", Required = false, HelpText = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain.")]
         public FileInfo WasmJavascriptEngine { get; set; }
 

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -281,6 +281,8 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig)
                 baseJob = baseJob.WithPlatform(options.Platform.Value);
             if (options.RunOncePerIteration)
                 baseJob = baseJob.RunOncePerIteration();
+            if (options.MemoryRandomization)
+                baseJob = baseJob.WithMemoryRandomization();
 
             if (options.EnvironmentVariables.Any())
             {

src/BenchmarkDotNet/Engines/Engine.cs

@@ -1,7 +1,9 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
@@ -33,16 +35,18 @@ public class Engine : IEngine
         [PublicAPI] public string BenchmarkName { get; }
 
         private IClock Clock { get; }
-        private bool ForceAllocations { get; }
+        private bool ForceGcCleanups { get; }
         private int UnrollFactor { get; }
         private RunStrategy Strategy { get; }
         private bool EvaluateOverhead { get; }
         private int InvocationCount { get; }
+        private bool MemoryRandomization { get; }
 
         private readonly EnginePilotStage pilotStage;
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineActualStage actualStage;
         private readonly bool includeExtraStats;
+        private readonly Random random;
 
         internal Engine(
             IHost host,
@@ -70,15 +74,18 @@ internal Engine(
             Resolver = resolver;
 
             Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver);
-            ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver);
+            ForceGcCleanups = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver);
             UnrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, Resolver);
             Strategy = targetJob.ResolveValue(RunMode.RunStrategyCharacteristic, Resolver);
             EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver);
             InvocationCount = targetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver);
+            MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver);
 
             warmupStage = new EngineWarmupStage(this);
             pilotStage = new EnginePilotStage(this);
             actualStage = new EngineActualStage(this);
+
+            random = new Random(12345); // we are using constant seed to try to get repeatable results
         }
 
         public void Dispose()
@@ -146,6 +153,7 @@ public Measurement RunIteration(IterationData data)
             int unrollFactor = data.UnrollFactor;
             long totalOperations = invokeCount * OperationsPerInvoke;
             bool isOverhead = data.IterationMode == IterationMode.Overhead;
+            bool randomizeMemory = !isOverhead && MemoryRandomization;
             var action = isOverhead ? OverheadAction : WorkloadAction;
 
             if (!isOverhead)
@@ -156,6 +164,8 @@ public Measurement RunIteration(IterationData data)
             if (EngineEventSource.Log.IsEnabled())
                 EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations);
 
+            Span<byte> stackMemory = randomizeMemory ? stackalloc byte[random.Next(32)] : Span<byte>.Empty;
+
             // Measure
             var clock = Clock.Start();
             action(invokeCount / unrollFactor);
@@ -167,12 +177,17 @@ public Measurement RunIteration(IterationData data)
             if (!isOverhead)
                 IterationCleanupAction();
 
+            if (randomizeMemory)
+                RandomizeManagedHeapMemory();
+
             GcCollect();
 
             // Results
             var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds());
             WriteLine(measurement.ToString());
 
+            Consume(stackMemory);
+
             return measurement;
         }
 
@@ -201,9 +216,30 @@ public Measurement RunIteration(IterationData data)
             return (gcStats, threadingStats);
         }
 
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void Consume(in Span<byte> _) { }
+
+        private void RandomizeManagedHeapMemory()
+        {
+            // invoke global cleanup before global setup
+            GlobalCleanupAction?.Invoke();
+
+            var gen0object = new byte[random.Next(32)];
+            var lohObject = new byte[85 * 1024 + random.Next(32)];
+
+            // we expect the key allocations to happen in global setup (not ctor)
+            // so we call it while keeping the random-size objects alive
+            GlobalSetupAction?.Invoke();
+
+            GC.KeepAlive(gen0object);
+            GC.KeepAlive(lohObject);
+
+            // we don't enforce GC.Collects here as engine does it later anyway
+        }
+
         private void GcCollect()
         {
-            if (!ForceAllocations)
+            if (!ForceGcCleanups)
                 return;
 
             ForceGcCollect();

src/BenchmarkDotNet/Engines/EngineResolver.cs

@@ -33,8 +33,15 @@ private EngineResolver()
             Register(AccuracyMode.MinIterationTimeCharacteristic, () => TimeInterval.Millisecond * 500);
             Register(AccuracyMode.MinInvokeCountCharacteristic, () => 4);
             Register(AccuracyMode.EvaluateOverheadCharacteristic, () => true);
+            Register(RunMode.MemoryRandomizationCharacteristic, () => false);
             Register(AccuracyMode.OutlierModeCharacteristic, job =>
             {
+                // if Memory Randomization was enabled and the benchmark is truly multimodal
+                // removing outliers could remove some values that are not actually outliers
+                // see https://github.com/dotnet/BenchmarkDotNet/pull/1587#issue-516837573 for example
+                if (job.ResolveValue(RunMode.MemoryRandomizationCharacteristic, this))
+                    return OutlierMode.DontRemove;
+
                 var strategy = job.ResolveValue(RunMode.RunStrategyCharacteristic, this);
                 switch (strategy)
                 {

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -210,12 +210,17 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) =
         /// </summary>
         public static Job WithPowerPlan(this Job job, Guid powerPlanGuid) => job.WithCore(j => j.Environment.PowerPlanMode = powerPlanGuid);
 
-
         /// <summary>
         /// ensures that BenchmarkDotNet does not enforce any power plan
         /// </summary>
         public static Job DontEnforcePowerPlan(this Job job) => job.WithCore(j => j.Environment.PowerPlanMode = Guid.Empty);
 
+        /// <summary>
+        /// specifies whether Engine should allocate some random-sized memory between iterations
+        /// <remarks>it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration</remarks>
+        /// </summary>
+        public static Job WithMemoryRandomization(this Job job, bool enable = true) => job.WithCore(j => j.Run.MemoryRandomization = enable);
+
         // Infrastructure
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("This method will soon be removed, please start using .WithToolchain instead")]

src/BenchmarkDotNet/Jobs/RunMode.cs

@@ -21,6 +21,7 @@ public sealed class RunMode : JobMode<RunMode>
         public static readonly Characteristic<int> WarmupCountCharacteristic = CreateCharacteristic<int>(nameof(WarmupCount));
         public static readonly Characteristic<int> MinWarmupIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MinWarmupIterationCount));
         public static readonly Characteristic<int> MaxWarmupIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MaxWarmupIterationCount));
+        public static readonly Characteristic<bool> MemoryRandomizationCharacteristic = CreateCharacteristic<bool>(nameof(MemoryRandomization));
 
         public static readonly RunMode Dry = new RunMode(nameof(Dry))
         {
@@ -180,5 +181,15 @@ public int MaxWarmupIterationCount
             get { return MaxWarmupIterationCountCharacteristic[this]; }
             set { MaxWarmupIterationCountCharacteristic[this] = value; }
         }
+
+        /// <summary>
+        /// specifies whether Engine should allocate some random-sized memory between iterations
+        /// <remarks>it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration</remarks>
+        /// </summary>
+        public bool MemoryRandomization
+        {
+            get => MemoryRandomizationCharacteristic[this];
+            set => MemoryRandomizationCharacteristic[this] = value;
+        }
     }
 }