Added manual tests for actual work.

Author: timcassell

tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs

@@ -8,6 +8,7 @@
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.IntegrationTests.ManualRunning;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
@@ -48,15 +49,15 @@ private static IEnumerable<Type> EmptyBenchmarkTypes() =>
                 typeof(EmptyClass)
             };
 
-        public static IEnumerable<object[]> InProcessData()
+        public static IEnumerable<object[]> EmptyInProcessData()
         {
             foreach (var type in EmptyBenchmarkTypes())
             {
                 yield return new object[] { type };
             }
         }
 
-        public static IEnumerable<object[]> CoreData()
+        public static IEnumerable<object[]> EmptyCoreData()
         {
             foreach (var type in EmptyBenchmarkTypes())
             {
@@ -65,7 +66,7 @@ public static IEnumerable<object[]> CoreData()
             }
         }
 
-        public static IEnumerable<object[]> FrameworkData()
+        public static IEnumerable<object[]> EmptyFrameworkData()
         {
             foreach (var type in EmptyBenchmarkTypes())
             {
@@ -75,7 +76,7 @@ public static IEnumerable<object[]> FrameworkData()
         }
 
         [Theory]
-        [MemberData(nameof(InProcessData))]
+        [MemberData(nameof(EmptyInProcessData))]
         public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType)
         {
             AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
@@ -85,7 +86,7 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkTy
         }
 
         [TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)]
-        [MemberData(nameof(CoreData))]
+        [MemberData(nameof(EmptyCoreData))]
         public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
             AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
@@ -95,7 +96,7 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, R
         }
 
         [TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
-        [MemberData(nameof(FrameworkData))]
+        [MemberData(nameof(EmptyFrameworkData))]
         public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
             AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
@@ -129,57 +130,91 @@ private void AssertZeroResults(Type benchmarkType, IConfig config)
             }
         }
 
-        [Fact]
-        public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess()
+        private static IEnumerable<Type> NonEmptyBenchmarkTypes() =>
+            new[]
+            {
+                typeof(DifferentSizedStructs),
+                typeof(ActualWork)
+            };
+
+        public static IEnumerable<object[]> NonEmptyInProcessData()
+        {
+            foreach (var type in NonEmptyBenchmarkTypes())
+            {
+                yield return new object[] { type };
+            }
+        }
+
+        public static IEnumerable<object[]> NonEmptyCoreData()
+        {
+            foreach (var type in NonEmptyBenchmarkTypes())
+            {
+                yield return new object[] { type, RuntimeMoniker.Net70 };
+                yield return new object[] { type, RuntimeMoniker.Mono70 };
+            }
+        }
+
+        public static IEnumerable<object[]> NonEmptyFrameworkData()
+        {
+            foreach (var type in NonEmptyBenchmarkTypes())
+            {
+                yield return new object[] { type, RuntimeMoniker.Net462 };
+                yield return new object[] { type, RuntimeMoniker.Mono };
+            }
+        }
+
+        [Theory]
+        [MemberData(nameof(NonEmptyInProcessData))]
+        public void NonEmptyBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess(Type benchmarkType)
         {
-            AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
+            AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithToolchain(InProcessEmitToolchain.Instance)
                 ));
         }
 
         [TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)]
-        [InlineData(RuntimeMoniker.Net70)]
-        [InlineData(RuntimeMoniker.Mono70)]
-        public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker)
+        [MemberData(nameof(NonEmptyCoreData))]
+        public void NonEmptyBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
-            AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
+            AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithRuntime(runtimeMoniker.GetRuntime())
                 ));
         }
 
         [TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
-        [InlineData(RuntimeMoniker.Net462)]
-        [InlineData(RuntimeMoniker.Mono)]
-        public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker)
+        [MemberData(nameof(NonEmptyFrameworkData))]
+        public void NonEmptyBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
-            AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
+            AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithRuntime(runtimeMoniker.GetRuntime())
                 ));
         }
 
-        private void AssertDifferentSizedStructsResults(IConfig config)
+        private void AssertNonZeroResults(Type benchmarkType, IConfig config)
         {
-            var summary = CanExecute<DifferentSizedStructs>(config
+            var summary = CanExecute(benchmarkType, config
                 .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
                 .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
             );
 
             var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
-            var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
 
             foreach (var report in summary.Reports)
             {
                 var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
                 var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
 
-                bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
+                // We use the default threshold here rather than using cpu resolution,
+                // because modern cpus can execute multiple instructions per clock cycle,
+                // resulting in measurements greater than 0 but less than 1 clock cycle.
+                // (example: Intel Core i9-9880H CPU 2.30GHz reports 0.2852 ns for `_field++;`)
+                bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements);
                 Assert.False(isZero, $"Actual time was 0.");
 
-                isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
-                Assert.True(isZero, "Overhead took more time than workload.");
+                // We don't need to check if overhead was greater. It's guaranteed less if the actual time was not zero.
 
                 Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
             }
@@ -209,13 +244,25 @@ public struct Struct128
                     l9, l10, l11, l12,
                     l13, l14, l15, l16;
     }
+}
+
+public class DifferentSizedStructs
+{
+    [Benchmark] public Struct16 Struct16() => default;
+    [Benchmark] public Struct32 Struct32() => default;
+    [Benchmark] public Struct64 Struct64() => default;
+    [Benchmark] public Struct128 Struct128() => default;
+}
+
+public class ActualWork
+{
+    public int _field;
 
-    public class DifferentSizedStructs
+    [Benchmark]
+    public void IncrementField()
     {
-        [Benchmark] public Struct16 Struct16() => default;
-        [Benchmark] public Struct32 Struct32() => default;
-        [Benchmark] public Struct64 Struct64() => default;
-        [Benchmark] public Struct128 Struct128() => default;
+        _field++;
+        _field++;
     }
 }