Added optional includeSurvived flag on MemoryDiagnoserAttribute to capture the total application memory after the benchmark is done.

Need some way to measure the memory before a benchmark is run to subtract away the memory that is consumed by the benchmark runner itself.

Author: timcassell

src/BenchmarkDotNet/Attributes/MemoryDiagnoserAttribute.cs

@@ -9,9 +9,9 @@ public class MemoryDiagnoserAttribute : Attribute, IConfigSource
     {
         public IConfig Config { get; }
 
-        public MemoryDiagnoserAttribute()
+        public MemoryDiagnoserAttribute(bool includeSurvived = false)
         {
-            Config = ManualConfig.CreateEmpty().AddDiagnoser(MemoryDiagnoser.Default);
+            Config = ManualConfig.CreateEmpty().AddDiagnoser(includeSurvived ? MemoryDiagnoser.WithSurvived : MemoryDiagnoser.Default);
         }
     }
 }

src/BenchmarkDotNet/Code/CodeGenerator.cs

@@ -63,6 +63,7 @@ internal static string Generate(BuildPartition buildPartition)
                     .Replace("$PassArguments$", passArguments)
                     .Replace("$EngineFactoryType$", GetEngineFactoryTypeName(benchmark))
                     .Replace("$MeasureExtraStats$", buildInfo.Config.HasExtraStatsDiagnoser() ? "true" : "false")
+                    .Replace("$MeasureSurvivedMemory$", buildInfo.Config.HasSurvivedMemoryDiagnoser() ? "true" : "false")
                     .Replace("$DisassemblerEntryMethodName$", DisassemblerConstants.DisassemblerEntryMethodName)
                     .Replace("$WorkloadMethodCall$", provider.GetWorkloadMethodCall(passArguments)).ToString();
 

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -92,7 +92,9 @@ internal ImmutableConfig(
         public IAnalyser GetCompositeAnalyser() => new CompositeAnalyser(analysers);
         public IDiagnoser GetCompositeDiagnoser() => new CompositeDiagnoser(diagnosers);
 
-        public bool HasMemoryDiagnoser() => diagnosers.Contains(MemoryDiagnoser.Default);
+        public bool HasMemoryDiagnoser() => diagnosers.Any(diagnoser => diagnoser is MemoryDiagnoser);
+
+        public bool HasSurvivedMemoryDiagnoser() => diagnosers.Contains(MemoryDiagnoser.WithSurvived);
 
         public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default);
 

src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs

@@ -15,9 +15,15 @@ public class MemoryDiagnoser : IDiagnoser
     {
         private const string DiagnoserId = nameof(MemoryDiagnoser);
 
-        public static readonly MemoryDiagnoser Default = new MemoryDiagnoser();
+        public static readonly MemoryDiagnoser Default = new MemoryDiagnoser(false);
+        public static readonly MemoryDiagnoser WithSurvived = new MemoryDiagnoser(true);
 
-        private MemoryDiagnoser() { } // we want to have only a single instance of MemoryDiagnoser
+        private MemoryDiagnoser(bool includeSurvived)
+        {
+            IncludeSurvived = includeSurvived;
+        }
+
+        public bool IncludeSurvived { get; }
 
         public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
 
@@ -37,6 +43,10 @@ public IEnumerable<Metric> ProcessResults(DiagnoserResults diagnoserResults)
             yield return new Metric(GarbageCollectionsMetricDescriptor.Gen1, diagnoserResults.GcStats.Gen1Collections / (double)diagnoserResults.GcStats.TotalOperations * 1000);
             yield return new Metric(GarbageCollectionsMetricDescriptor.Gen2, diagnoserResults.GcStats.Gen2Collections / (double)diagnoserResults.GcStats.TotalOperations * 1000);
             yield return new Metric(AllocatedMemoryMetricDescriptor.Instance, diagnoserResults.GcStats.BytesAllocatedPerOperation);
+            if (IncludeSurvived)
+            {
+                yield return new Metric(SurvivedMemoryMetricDescriptor.Instance, diagnoserResults.GcStats.SurvivedBytes);
+            }
         }
 
         private class AllocatedMemoryMetricDescriptor : IMetricDescriptor
@@ -52,6 +62,19 @@ private class AllocatedMemoryMetricDescriptor : IMetricDescriptor
             public bool TheGreaterTheBetter => false;
         }
 
+        private class SurvivedMemoryMetricDescriptor : IMetricDescriptor
+        {
+            internal static readonly IMetricDescriptor Instance = new SurvivedMemoryMetricDescriptor();
+
+            public string Id => "Survived Memory";
+            public string DisplayName => "Survived";
+            public string Legend => "Total application memory consumed after all operations, including Cleanups (managed only, inclusive, 1KB = 1024B)";
+            public string NumberFormat => "N0";
+            public UnitType UnitType => UnitType.Size;
+            public string Unit => SizeUnit.B.Name;
+            public bool TheGreaterTheBetter => false;
+        }
+
         private class GarbageCollectionsMetricDescriptor : IMetricDescriptor
         {
             internal static readonly IMetricDescriptor Gen0 = new GarbageCollectionsMetricDescriptor(0);

src/BenchmarkDotNet/Engines/Engine.cs

@@ -42,14 +42,14 @@ public class Engine : IEngine
         private readonly EnginePilotStage pilotStage;
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineActualStage actualStage;
-        private readonly bool includeExtraStats;
+        private readonly bool includeExtraStats, includeSurvivedMemory;
 
         internal Engine(
             IHost host,
             IResolver resolver,
             Action dummy1Action, Action dummy2Action, Action dummy3Action, Action<long> overheadAction, Action<long> workloadAction, Job targetJob,
             Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke,
-            bool includeExtraStats, string benchmarkName)
+            bool includeExtraStats, bool includeSurvivedMemory, string benchmarkName)
         {
 
             Host = host;
@@ -66,6 +66,7 @@ internal Engine(
             OperationsPerInvoke = operationsPerInvoke;
             this.includeExtraStats = includeExtraStats;
             BenchmarkName = benchmarkName;
+            this.includeSurvivedMemory = includeSurvivedMemory;
 
             Resolver = resolver;
 
@@ -135,8 +136,8 @@ public RunResults Run()
                 EngineEventSource.Log.BenchmarkStop(BenchmarkName);
 
             var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver);
-
-            return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats);
+            
+            return new RunResults(idle, main, outlierMode, workGcHasDone.WithSurvivedBytes(includeSurvivedMemory), threadingStats);
         }
 
         public Measurement RunIteration(IterationData data)

src/BenchmarkDotNet/Engines/EngineFactory.cs

@@ -119,6 +119,7 @@ private static Engine CreateEngine(EngineParameters engineParameters, Job job, A
                 engineParameters.IterationCleanupAction,
                 engineParameters.OperationsPerInvoke,
                 engineParameters.MeasureExtraStats,
+                engineParameters.MeasureSurvivedMemory,
                 engineParameters.BenchmarkName);
     }
 }

src/BenchmarkDotNet/Engines/EngineParameters.cs

@@ -27,6 +27,8 @@ public class EngineParameters
         public Action IterationCleanupAction { get; set; }
         public bool MeasureExtraStats { get; set; }
 
+        public bool MeasureSurvivedMemory { get; set; }
+
         [PublicAPI] public string BenchmarkName { get;  set; }
 
         public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting();

src/BenchmarkDotNet/Engines/GcStats.cs

@@ -15,15 +15,16 @@ public struct GcStats : IEquatable<GcStats>
         private static readonly Func<long> GetAllocatedBytesForCurrentThreadDelegate = CreateGetAllocatedBytesForCurrentThreadDelegate();
         private static readonly Func<bool, long> GetTotalAllocatedBytesDelegate = CreateGetTotalAllocatedBytesDelegate();
 
-        public static readonly GcStats Empty = new GcStats(0, 0, 0, 0, 0);
+        public static readonly GcStats Empty = new GcStats(0, 0, 0, 0, 0, 0);
 
-        private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long allocatedBytes, long totalOperations)
+        private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long allocatedBytes, long totalOperations, long survivedBytes)
         {
             Gen0Collections = gen0Collections;
             Gen1Collections = gen1Collections;
             Gen2Collections = gen2Collections;
             AllocatedBytes = allocatedBytes;
             TotalOperations = totalOperations;
+            SurvivedBytes = survivedBytes;
         }
 
         // did not use array here just to avoid heap allocation
@@ -37,6 +38,7 @@ private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, l
         private long AllocatedBytes { get; }
 
         public long TotalOperations { get; }
+        public long SurvivedBytes { get; }
 
         public long BytesAllocatedPerOperation
         {
@@ -60,7 +62,8 @@ public long BytesAllocatedPerOperation
                 left.Gen1Collections + right.Gen1Collections,
                 left.Gen2Collections + right.Gen2Collections,
                 left.AllocatedBytes + right.AllocatedBytes,
-                left.TotalOperations + right.TotalOperations);
+                left.TotalOperations + right.TotalOperations,
+                left.SurvivedBytes + right.SurvivedBytes);
         }
 
         public static GcStats operator -(GcStats left, GcStats right)
@@ -70,11 +73,15 @@ public long BytesAllocatedPerOperation
                 Math.Max(0, left.Gen1Collections - right.Gen1Collections),
                 Math.Max(0, left.Gen2Collections - right.Gen2Collections),
                 Math.Max(0, left.AllocatedBytes - right.AllocatedBytes),
-                Math.Max(0, left.TotalOperations - right.TotalOperations));
+                Math.Max(0, left.TotalOperations - right.TotalOperations),
+                Math.Max(0, left.SurvivedBytes - right.SurvivedBytes));
         }
 
         public GcStats WithTotalOperations(long totalOperationsCount)
-            => this + new GcStats(0, 0, 0, 0, totalOperationsCount);
+            => this + new GcStats(0, 0, 0, 0, totalOperationsCount, 0);
+
+        public GcStats WithSurvivedBytes(bool getBytes)
+            => this + new GcStats(0, 0, 0, 0, 0, GetTotalBytes(getBytes));
 
         public int GetCollectionsCount(int generation)
         {
@@ -112,6 +119,7 @@ public static GcStats ReadInitial()
                 GC.CollectionCount(1),
                 GC.CollectionCount(2),
                 allocatedBytes,
+                0,
                 0);
         }
 
@@ -125,12 +133,31 @@ public static GcStats ReadFinal()
                 // this will force GC.Collect, so we want to do this after collecting collections counts
                 // to exclude this single full forced collection from results
                 GetAllocatedBytes(),
+                0,
                 0);
         }
 
         [PublicAPI]
         public static GcStats FromForced(int forcedFullGarbageCollections)
-            => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0);
+            => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0, 0);
+
+        private static long GetTotalBytes(bool actual)
+        {
+            if (!actual)
+                return 0;
+
+            if (RuntimeInformation.IsFullFramework) // it can be a .NET app consuming our .NET Standard package
+            {
+                AppDomain.MonitoringIsEnabled = true;
+
+                // Enforce GC.Collect here just to make sure we get accurate results
+                GC.Collect();
+                return AppDomain.CurrentDomain.MonitoringSurvivedMemorySize;
+            }
+
+            GC.Collect();
+            return GC.GetTotalMemory(true);
+        }
 
         private static long GetAllocatedBytes()
         {
@@ -171,7 +198,7 @@ private static Func<bool, long> CreateGetTotalAllocatedBytesDelegate()
         }
 
         public string ToOutputLine()
-            => $"{ResultsLinePrefix} {Gen0Collections} {Gen1Collections} {Gen2Collections} {AllocatedBytes} {TotalOperations}";
+            => $"{ResultsLinePrefix} {Gen0Collections} {Gen1Collections} {Gen2Collections} {AllocatedBytes} {TotalOperations} {SurvivedBytes}";
 
         public static GcStats Parse(string line)
         {
@@ -183,12 +210,13 @@ public static GcStats Parse(string line)
                 || !int.TryParse(measurementSplit[1], out int gen1)
                 || !int.TryParse(measurementSplit[2], out int gen2)
                 || !long.TryParse(measurementSplit[3], out long allocatedBytes)
-                || !long.TryParse(measurementSplit[4], out long totalOperationsCount))
+                || !long.TryParse(measurementSplit[4], out long totalOperationsCount)
+                || !long.TryParse(measurementSplit[5], out long survivedBytes))
             {
                 throw new NotSupportedException("Invalid string");
             }
 
-            return new GcStats(gen0, gen1, gen2, allocatedBytes, totalOperationsCount);
+            return new GcStats(gen0, gen1, gen2, allocatedBytes, totalOperationsCount, survivedBytes);
         }
 
         public override string ToString() => ToOutputLine();
@@ -222,7 +250,13 @@ private static long CalculateAllocationQuantumSize()
             return result;
         }
 
-        public bool Equals(GcStats other) => Gen0Collections == other.Gen0Collections && Gen1Collections == other.Gen1Collections && Gen2Collections == other.Gen2Collections && AllocatedBytes == other.AllocatedBytes && TotalOperations == other.TotalOperations;
+        public bool Equals(GcStats other) =>
+            Gen0Collections == other.Gen0Collections
+            && Gen1Collections == other.Gen1Collections
+            && Gen2Collections == other.Gen2Collections
+            && AllocatedBytes == other.AllocatedBytes
+            && TotalOperations == other.TotalOperations
+            && SurvivedBytes == other.SurvivedBytes;
 
         public override bool Equals(object obj) => obj is GcStats other && Equals(other);
 

src/BenchmarkDotNet/Exporters/Csv/CsvMeasurementsExporter.cs

@@ -71,6 +71,10 @@ private static MeasurementColumn[] GetColumns(Summary summary)
                 new MeasurementColumn("Gen_2", (_, report, __) => report.GcStats.Gen2Collections.ToString(summary.GetCultureInfo())),
                 new MeasurementColumn("Allocated_Bytes", (_, report, __) => report.GcStats.BytesAllocatedPerOperation.ToString(summary.GetCultureInfo()))
             };
+            if (summary.BenchmarksCases.Any(benchmark => benchmark.Config.HasSurvivedMemoryDiagnoser()))
+            {
+                columns.Add(new MeasurementColumn("Survived_Bytes", (_, report, __) => report.GcStats.SurvivedBytes.ToString(summary.GetCultureInfo())));
+            }
 
             return columns.ToArray();
         }

src/BenchmarkDotNet/Templates/BenchmarkType.txt

@@ -38,6 +38,7 @@
                 TargetJob = job,
                 OperationsPerInvoke = $OperationsPerInvoke$,
                 MeasureExtraStats = $MeasureExtraStats$,
+                MeasureSurvivedMemory = $MeasureSurvivedMemory$,
                 BenchmarkName = benchmarkName
             };