From 8543aec21a77277d542c5cd9eeb0f03a3fa4ff37 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 03:56:43 -0400 Subject: [PATCH 1/9] Added manual tests for expected results. --- .../Properties/AssemblyInfo.cs | 2 + ...tNet.IntegrationTests.ManualRunning.csproj | 2 +- .../ExpectedBenchmarkResultsTests.cs | 220 ++++++++++++++++++ .../BenchmarkTestExecutor.cs | 2 +- 4 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index 21e7364a24..596c220645 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -14,9 +14,11 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] #else [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")] +[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")] #endif \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index 77e3aa5cff..b0259a941e 100755 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -16,7 +16,6 @@ - @@ -26,6 +25,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs new file mode 100644 index 0000000000..91782cd1d9 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests.ManualRunning +{ + public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor + { + // NativeAot takes a long time to build, so not including it in these tests. + // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. + + public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } + + private static IEnumerable EmptyBenchmarkTypes() => + new[] + { + typeof(EmptyVoid), + typeof(EmptyByte), + typeof(EmptySByte), + typeof(EmptyShort), + typeof(EmptyUShort), + typeof(EmptyChar), + typeof(EmptyInt32), + typeof(EmptyUInt32), + typeof(EmptyInt64), + typeof(EmptyUInt64), + typeof(EmptyIntPtr), + typeof(EmptyUIntPtr), + typeof(EmptyVoidPointer), + typeof(EmptyClass) + }; + + public static IEnumerable InProcessData() + { + foreach (var type in EmptyBenchmarkTypes()) + { + yield return new object[] { type }; + } + } + + public static IEnumerable CoreData() + { + foreach (var type in EmptyBenchmarkTypes()) + { + yield return new object[] { type, RuntimeMoniker.Net70 }; + yield return new object[] { type, RuntimeMoniker.Mono70 }; + } + } + + public static IEnumerable FrameworkData() + { + foreach (var type in EmptyBenchmarkTypes()) + { + yield return new object[] { type, RuntimeMoniker.Net462 }; + yield return new object[] { type, RuntimeMoniker.Mono }; + } + } + + [Theory] + [MemberData(nameof(InProcessData))] + public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType) + { + AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithToolchain(InProcessEmitToolchain.Instance) + )); + } + + [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] + [MemberData(nameof(CoreData))] + public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker) + { + AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] + [MemberData(nameof(FrameworkData))] + public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker) + { + AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + private void AssertZeroResults(Type benchmarkType, IConfig config) + { + var summary = CanExecute(benchmarkType, config + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) + .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) + ); + + var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz(); + + foreach (var report in summary.Reports) + { + Assert.True(cpuGhz * report.ResultStatistics.Mean < 1, $"Actual time was greater than 1 clock cycle."); + + var overheadTime = report.AllMeasurements + .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) + .Select(m => m.GetAverageTime().Nanoseconds) + .Average(); + + var workloadTime = report.AllMeasurements + .Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual) + .Select(m => m.GetAverageTime().Nanoseconds) + .Average(); + + // Allow for 1 cpu cycle variance + Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); + + Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); + } + } + + [Fact] + public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() + { + AssertLargeStructResults(ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithToolchain(InProcessEmitToolchain.Instance) + )); + } + + [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] + [InlineData(RuntimeMoniker.Net70)] + [InlineData(RuntimeMoniker.Mono70)] + public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker) + { + AssertLargeStructResults(ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] + [InlineData(RuntimeMoniker.Net462)] + [InlineData(RuntimeMoniker.Mono)] + public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker) + { + AssertLargeStructResults(ManualConfig.CreateEmpty() + .AddJob(Job.Default + .WithRuntime(runtimeMoniker.GetRuntime()) + )); + } + + private void AssertLargeStructResults(IConfig config) + { + var summary = CanExecute(config + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) + .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) + ); + + var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz(); + + foreach (var report in summary.Reports) + { + Assert.True(cpuGhz * report.ResultStatistics.Mean >= 1, $"Actual time was less than 1 clock cycle."); + + var overheadTime = report.AllMeasurements + .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) + .Select(m => m.GetAverageTime().Nanoseconds) + .Average(); + + var workloadTime = report.AllMeasurements + .Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual) + .Select(m => m.GetAverageTime().Nanoseconds) + .Average(); + + // Allow for 1 cpu cycle variance + Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); + + Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); + } + } + } + + public class LargeStruct + { + public struct Struct + { + // 128 bits + public long l1, l2, l3, l4, + l5, l6, l7, l8, + l9, l10, l11, l12, + l13, l14, l15, l16; + } + + [Benchmark] public Struct Benchmark() => default; + } +} + +public class EmptyVoid { [Benchmark] public void Benchmark() { } } +public class EmptyByte { [Benchmark] public byte Benchmark() => default; } +public class EmptySByte { [Benchmark] public sbyte Benchmark() => default; } +public class EmptyShort { [Benchmark] public short Benchmark() => default; } +public class EmptyUShort { [Benchmark] public ushort Benchmark() => default; } +public class EmptyChar { [Benchmark] public char Benchmark() => default; } +public class EmptyInt32 { [Benchmark] public int Benchmark() => default; } +public class EmptyUInt32 { [Benchmark] public uint Benchmark() => default; } +public class EmptyInt64 { [Benchmark] public long Benchmark() => default; } +public class EmptyUInt64 { [Benchmark] public ulong Benchmark() => default; } +public class EmptyIntPtr { [Benchmark] public IntPtr Benchmark() => default; } +public class EmptyUIntPtr { [Benchmark] public UIntPtr Benchmark() => default; } +public class EmptyVoidPointer { [Benchmark] public unsafe void* Benchmark() => default; } +public class EmptyClass { [Benchmark] public object Class() => default; } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs index 032e2643d8..1171b0f014 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs @@ -50,7 +50,7 @@ public Reports.Summary CanExecute(IConfig config = null, bool fullVa /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) /// The summary from the benchmark run - protected Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true) + public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true) { // Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier) if (config == null) From 6dfe6f47c0918c7881760f322e9dcc17bbe9ba55 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 07:06:56 -0400 Subject: [PATCH 2/9] Use a fallback value for cpu resolution. Use ZeroMeasurementHelper. --- .../ExpectedBenchmarkResultsTests.cs | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 91782cd1d9..ad480868dd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; @@ -10,6 +11,7 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer.Horology; using Xunit; using Xunit.Abstractions; @@ -20,6 +22,8 @@ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor // NativeAot takes a long time to build, so not including it in these tests. // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. + private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d); + public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } private static IEnumerable EmptyBenchmarkTypes() => @@ -104,21 +108,25 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz(); + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var cpuGhz = cpuResolution.ToFrequency().ToGHz(); foreach (var report in summary.Reports) { - Assert.True(cpuGhz * report.ResultStatistics.Mean < 1, $"Actual time was greater than 1 clock cycle."); - - var overheadTime = report.AllMeasurements + var workloadTimes = report.AllMeasurements .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) .Select(m => m.GetAverageTime().Nanoseconds) - .Average(); - - var workloadTime = report.AllMeasurements - .Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual) + .ToArray(); + var overheadTimes = report.AllMeasurements + .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) .Select(m => m.GetAverageTime().Nanoseconds) - .Average(); + .ToArray(); + + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadTimes, overheadTimes); + Assert.True(isZero, $"Actual time was not 0."); + + var workloadTime = workloadTimes.Average(); + var overheadTime = overheadTimes.Average(); // Allow for 1 cpu cycle variance Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); @@ -165,21 +173,25 @@ private void AssertLargeStructResults(IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz(); + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var cpuGhz = cpuResolution.ToFrequency().ToGHz(); foreach (var report in summary.Reports) { - Assert.True(cpuGhz * report.ResultStatistics.Mean >= 1, $"Actual time was less than 1 clock cycle."); - - var overheadTime = report.AllMeasurements + var workloadTimes = report.AllMeasurements .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) .Select(m => m.GetAverageTime().Nanoseconds) - .Average(); - - var workloadTime = report.AllMeasurements - .Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual) + .ToArray(); + var overheadTimes = report.AllMeasurements + .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) .Select(m => m.GetAverageTime().Nanoseconds) - .Average(); + .ToArray(); + + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadTimes, overheadTimes); + Assert.False(isZero, $"Actual time was 0."); + + var workloadTime = workloadTimes.Average(); + var overheadTime = overheadTimes.Average(); // Allow for 1 cpu cycle variance Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); From e0959b94087adbd876ceb14012c6b33f0e6bb892 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 07:33:19 -0400 Subject: [PATCH 3/9] Add multiple struct sizes. --- .../ExpectedBenchmarkResultsTests.cs | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index ad480868dd..3a0a180d13 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -136,9 +136,9 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) } [Fact] - public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() + public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() { - AssertLargeStructResults(ManualConfig.CreateEmpty() + AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() .AddJob(Job.Default .WithToolchain(InProcessEmitToolchain.Instance) )); @@ -147,9 +147,9 @@ public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] [InlineData(RuntimeMoniker.Net70)] [InlineData(RuntimeMoniker.Mono70)] - public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker) + public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker) { - AssertLargeStructResults(ManualConfig.CreateEmpty() + AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() .AddJob(Job.Default .WithRuntime(runtimeMoniker.GetRuntime()) )); @@ -158,17 +158,17 @@ public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(Runtime [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] [InlineData(RuntimeMoniker.Net462)] [InlineData(RuntimeMoniker.Mono)] - public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker) + public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker) { - AssertLargeStructResults(ManualConfig.CreateEmpty() + AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty() .AddJob(Job.Default .WithRuntime(runtimeMoniker.GetRuntime()) )); } - private void AssertLargeStructResults(IConfig config) + private void AssertDifferentSizedStructsResults(IConfig config) { - var summary = CanExecute(config + var summary = CanExecute(config .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); @@ -201,18 +201,36 @@ private void AssertLargeStructResults(IConfig config) } } - public class LargeStruct + public struct Struct16 { - public struct Struct - { - // 128 bits - public long l1, l2, l3, l4, - l5, l6, l7, l8, - l9, l10, l11, l12, - l13, l14, l15, l16; - } + public long l1, l2; + } - [Benchmark] public Struct Benchmark() => default; + public struct Struct32 + { + public long l1, l2, l3, l4; + } + + public struct Struct64 + { + public long l1, l2, l3, l4, + l5, l6, l7, l8; + } + + public struct Struct128 + { + public long l1, l2, l3, l4, + l5, l6, l7, l8, + l9, l10, l11, l12, + l13, l14, l15, l16; + } + + public class DifferentSizedStructs + { + [Benchmark] public Struct16 Struct16() => default; + [Benchmark] public Struct32 Struct32() => default; + [Benchmark] public Struct64 Struct64() => default; + [Benchmark] public Struct128 Struct128() => default; } } From 78ce31d2dac9ed684fa0991367fd1bb6b533ddab Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 07:41:47 -0400 Subject: [PATCH 4/9] Use same code as ZeroMeasurementAnalyser. --- .../ExpectedBenchmarkResultsTests.cs | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 3a0a180d13..e8d4662333 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -113,20 +114,14 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) foreach (var report in summary.Reports) { - var workloadTimes = report.AllMeasurements - .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) - .Select(m => m.GetAverageTime().Nanoseconds) - .ToArray(); - var overheadTimes = report.AllMeasurements - .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) - .Select(m => m.GetAverageTime().Nanoseconds) - .ToArray(); - - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadTimes, overheadTimes); + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); Assert.True(isZero, $"Actual time was not 0."); - var workloadTime = workloadTimes.Average(); - var overheadTime = overheadTimes.Average(); + var workloadTime = workloadMeasurements.Average(); + var overheadTime = overheadMeasurements.Average(); // Allow for 1 cpu cycle variance Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); @@ -178,20 +173,14 @@ private void AssertDifferentSizedStructsResults(IConfig config) foreach (var report in summary.Reports) { - var workloadTimes = report.AllMeasurements - .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) - .Select(m => m.GetAverageTime().Nanoseconds) - .ToArray(); - var overheadTimes = report.AllMeasurements - .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) - .Select(m => m.GetAverageTime().Nanoseconds) - .ToArray(); - - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadTimes, overheadTimes); + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); Assert.False(isZero, $"Actual time was 0."); - var workloadTime = workloadTimes.Average(); - var overheadTime = overheadTimes.Average(); + var workloadTime = workloadMeasurements.Average(); + var overheadTime = overheadMeasurements.Average(); // Allow for 1 cpu cycle variance Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); From 7f163c86bfc92789ab612f9d7ae4a3d8f3daa3a5 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 08:14:52 -0400 Subject: [PATCH 5/9] Use ZeroMeasurementHelper for both checks. --- .../ExpectedBenchmarkResultsTests.cs | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index e8d4662333..3e6466b45d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -8,11 +8,9 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; -using Perfolizer.Horology; using Xunit; using Xunit.Abstractions; @@ -23,8 +21,6 @@ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor // NativeAot takes a long time to build, so not including it in these tests. // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. - private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d); - public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } private static IEnumerable EmptyBenchmarkTypes() => @@ -109,9 +105,6 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; - var cpuGhz = cpuResolution.ToFrequency().ToGHz(); - foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); @@ -120,11 +113,8 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); Assert.True(isZero, $"Actual time was not 0."); - var workloadTime = workloadMeasurements.Average(); - var overheadTime = overheadMeasurements.Average(); - - // Allow for 1 cpu cycle variance - Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements); + Assert.True(isZero, "Overhead took more time than workload."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } @@ -168,9 +158,6 @@ private void AssertDifferentSizedStructsResults(IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; - var cpuGhz = cpuResolution.ToFrequency().ToGHz(); - foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); @@ -179,11 +166,8 @@ private void AssertDifferentSizedStructsResults(IConfig config) bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); Assert.False(isZero, $"Actual time was 0."); - var workloadTime = workloadMeasurements.Average(); - var overheadTime = overheadMeasurements.Average(); - - // Allow for 1 cpu cycle variance - Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements); + Assert.True(isZero, "Overhead took more time than workload."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } From a6a462509e01aa6bce35442aed10debf99da3316 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 20:18:55 -0400 Subject: [PATCH 6/9] Set benchmark process priority to realtime. --- .../ExpectedBenchmarkResultsTests.cs | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 3e6466b45d..0e59568a69 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; @@ -198,7 +199,7 @@ public struct Struct128 l13, l14, l15, l16; } - public class DifferentSizedStructs + public class DifferentSizedStructs : RealTimeBenchmarks { [Benchmark] public Struct16 Struct16() => default; [Benchmark] public Struct32 Struct32() => default; @@ -207,17 +208,46 @@ public class DifferentSizedStructs } } -public class EmptyVoid { [Benchmark] public void Benchmark() { } } -public class EmptyByte { [Benchmark] public byte Benchmark() => default; } -public class EmptySByte { [Benchmark] public sbyte Benchmark() => default; } -public class EmptyShort { [Benchmark] public short Benchmark() => default; } -public class EmptyUShort { [Benchmark] public ushort Benchmark() => default; } -public class EmptyChar { [Benchmark] public char Benchmark() => default; } -public class EmptyInt32 { [Benchmark] public int Benchmark() => default; } -public class EmptyUInt32 { [Benchmark] public uint Benchmark() => default; } -public class EmptyInt64 { [Benchmark] public long Benchmark() => default; } -public class EmptyUInt64 { [Benchmark] public ulong Benchmark() => default; } -public class EmptyIntPtr { [Benchmark] public IntPtr Benchmark() => default; } -public class EmptyUIntPtr { [Benchmark] public UIntPtr Benchmark() => default; } -public class EmptyVoidPointer { [Benchmark] public unsafe void* Benchmark() => default; } -public class EmptyClass { [Benchmark] public object Class() => default; } \ No newline at end of file +public class RealTimeBenchmarks +{ + private Process process; + private ProcessPriorityClass oldPriority; + + [GlobalSetup] + public void Setup() + { + process = Process.GetCurrentProcess(); + try + { + oldPriority = process.PriorityClass; + // Requires admin mode. Makes the OS never give up CPU time for this process, so we can get more accurate timings. + process.PriorityClass = ProcessPriorityClass.RealTime; + } + catch (PlatformNotSupportedException) { } + } + + [GlobalCleanup] + public void Cleanup() + { + try + { + process.PriorityClass = oldPriority; + } + catch (PlatformNotSupportedException) { } + } +} + +public class EmptyVoid : RealTimeBenchmarks { [Benchmark] public void Benchmark() { } } +public class EmptyByte : RealTimeBenchmarks { [Benchmark] public byte Benchmark() => default; } +public class EmptySByte : RealTimeBenchmarks { [Benchmark] public sbyte Benchmark() => default; } +public class EmptyShort : RealTimeBenchmarks { [Benchmark] public short Benchmark() => default; } +public class EmptyUShort : RealTimeBenchmarks { [Benchmark] public ushort Benchmark() => default; } +public class EmptyChar : RealTimeBenchmarks { [Benchmark] public char Benchmark() => default; } +public class EmptyInt32 : RealTimeBenchmarks { [Benchmark] public int Benchmark() => default; } +public class EmptyUInt32 : RealTimeBenchmarks { [Benchmark] public uint Benchmark() => default; } +public class EmptyInt64 : RealTimeBenchmarks { [Benchmark] public long Benchmark() => default; } +public class EmptyUInt64 : RealTimeBenchmarks { [Benchmark] public ulong Benchmark() => default; } +public class EmptyIntPtr : RealTimeBenchmarks { [Benchmark] public IntPtr Benchmark() => default; } +public class EmptyUIntPtr : RealTimeBenchmarks { [Benchmark] public UIntPtr Benchmark() => default; } +public class EmptyVoidPointer : RealTimeBenchmarks { [Benchmark] public unsafe void* Benchmark() => default; } +public class EmptyClass : RealTimeBenchmarks { [Benchmark] public object Class() => default; } \ No newline at end of file From bb41ff3b9ba5ed493e4ff2805be79757439e370e Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 9 Jul 2023 23:29:57 -0400 Subject: [PATCH 7/9] Add threshold argument to ZeroMeasurementHelper. --- .../Analysers/ZeroMeasurementHelper.cs | 5 ++-- .../ExpectedBenchmarkResultsTests.cs | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs index 4ff5bf012a..d94087239c 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs @@ -1,4 +1,5 @@ using Perfolizer.Mathematics.SignificanceTesting; +using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Analysers { @@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh /// Checks distribution against Zero Measurement hypothesis in case of two samples /// /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead) + public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null) { if (workload.Length < 3 || overhead.Length < 3) return false; - return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected; + return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected; } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 0e59568a69..6b71e1e004 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -9,9 +9,12 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer.Horology; +using Perfolizer.Mathematics.Thresholds; using Xunit; using Xunit.Abstractions; @@ -22,6 +25,8 @@ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor // NativeAot takes a long time to build, so not including it in these tests. // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. + private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d); + public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } private static IEnumerable EmptyBenchmarkTypes() => @@ -102,19 +107,22 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkTy private void AssertZeroResults(Type benchmarkType, IConfig config) { var summary = CanExecute(benchmarkType, config - .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); Assert.True(isZero, $"Actual time was not 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements); + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); Assert.True(isZero, "Overhead took more time than workload."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); @@ -155,19 +163,22 @@ public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Fr private void AssertDifferentSizedStructsResults(IConfig config) { var summary = CanExecute(config - .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) + .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond)) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); + var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + foreach (var report in summary.Reports) { var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements); + bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); Assert.False(isZero, $"Actual time was 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements); + isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); Assert.True(isZero, "Overhead took more time than workload."); Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); From 3919a37bbca353c0e17f44c7f781a5395fa0c408 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 10 Jul 2023 00:00:33 -0400 Subject: [PATCH 8/9] Friendly benchmark names --- .../ExpectedBenchmarkResultsTests.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 6b71e1e004..4864cff64c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -248,17 +248,17 @@ public void Cleanup() } } -public class EmptyVoid : RealTimeBenchmarks { [Benchmark] public void Benchmark() { } } -public class EmptyByte : RealTimeBenchmarks { [Benchmark] public byte Benchmark() => default; } -public class EmptySByte : RealTimeBenchmarks { [Benchmark] public sbyte Benchmark() => default; } -public class EmptyShort : RealTimeBenchmarks { [Benchmark] public short Benchmark() => default; } -public class EmptyUShort : RealTimeBenchmarks { [Benchmark] public ushort Benchmark() => default; } -public class EmptyChar : RealTimeBenchmarks { [Benchmark] public char Benchmark() => default; } -public class EmptyInt32 : RealTimeBenchmarks { [Benchmark] public int Benchmark() => default; } -public class EmptyUInt32 : RealTimeBenchmarks { [Benchmark] public uint Benchmark() => default; } -public class EmptyInt64 : RealTimeBenchmarks { [Benchmark] public long Benchmark() => default; } -public class EmptyUInt64 : RealTimeBenchmarks { [Benchmark] public ulong Benchmark() => default; } -public class EmptyIntPtr : RealTimeBenchmarks { [Benchmark] public IntPtr Benchmark() => default; } -public class EmptyUIntPtr : RealTimeBenchmarks { [Benchmark] public UIntPtr Benchmark() => default; } -public class EmptyVoidPointer : RealTimeBenchmarks { [Benchmark] public unsafe void* Benchmark() => default; } +public class EmptyVoid : RealTimeBenchmarks { [Benchmark] public void Void() { } } +public class EmptyByte : RealTimeBenchmarks { [Benchmark] public byte Byte() => default; } +public class EmptySByte : RealTimeBenchmarks { [Benchmark] public sbyte SByte() => default; } +public class EmptyShort : RealTimeBenchmarks { [Benchmark] public short Short() => default; } +public class EmptyUShort : RealTimeBenchmarks { [Benchmark] public ushort UShort() => default; } +public class EmptyChar : RealTimeBenchmarks { [Benchmark] public char Char() => default; } +public class EmptyInt32 : RealTimeBenchmarks { [Benchmark] public int Int32() => default; } +public class EmptyUInt32 : RealTimeBenchmarks { [Benchmark] public uint UInt32() => default; } +public class EmptyInt64 : RealTimeBenchmarks { [Benchmark] public long Int64() => default; } +public class EmptyUInt64 : RealTimeBenchmarks { [Benchmark] public ulong UInt64() => default; } +public class EmptyIntPtr : RealTimeBenchmarks { [Benchmark] public IntPtr IntPtr() => default; } +public class EmptyUIntPtr : RealTimeBenchmarks { [Benchmark] public UIntPtr UIntPtr() => default; } +public class EmptyVoidPointer : RealTimeBenchmarks { [Benchmark] public unsafe void* VoidPointer() => default; } public class EmptyClass : RealTimeBenchmarks { [Benchmark] public object Class() => default; } \ No newline at end of file From 1dd6c84d384859ab1c400c79ab07199818bef9ac Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 10 Jul 2023 19:45:07 -0400 Subject: [PATCH 9/9] Removed RealTimeBenchmarks --- .../ExpectedBenchmarkResultsTests.cs | 59 +++++-------------- 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 4864cff64c..e36e43e0c3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -210,7 +210,7 @@ public struct Struct128 l13, l14, l15, l16; } - public class DifferentSizedStructs : RealTimeBenchmarks + public class DifferentSizedStructs { [Benchmark] public Struct16 Struct16() => default; [Benchmark] public Struct32 Struct32() => default; @@ -219,46 +219,17 @@ public class DifferentSizedStructs : RealTimeBenchmarks } } -public class RealTimeBenchmarks -{ - private Process process; - private ProcessPriorityClass oldPriority; - - [GlobalSetup] - public void Setup() - { - process = Process.GetCurrentProcess(); - try - { - oldPriority = process.PriorityClass; - // Requires admin mode. Makes the OS never give up CPU time for this process, so we can get more accurate timings. - process.PriorityClass = ProcessPriorityClass.RealTime; - } - catch (PlatformNotSupportedException) { } - } - - [GlobalCleanup] - public void Cleanup() - { - try - { - process.PriorityClass = oldPriority; - } - catch (PlatformNotSupportedException) { } - } -} - -public class EmptyVoid : RealTimeBenchmarks { [Benchmark] public void Void() { } } -public class EmptyByte : RealTimeBenchmarks { [Benchmark] public byte Byte() => default; } -public class EmptySByte : RealTimeBenchmarks { [Benchmark] public sbyte SByte() => default; } -public class EmptyShort : RealTimeBenchmarks { [Benchmark] public short Short() => default; } -public class EmptyUShort : RealTimeBenchmarks { [Benchmark] public ushort UShort() => default; } -public class EmptyChar : RealTimeBenchmarks { [Benchmark] public char Char() => default; } -public class EmptyInt32 : RealTimeBenchmarks { [Benchmark] public int Int32() => default; } -public class EmptyUInt32 : RealTimeBenchmarks { [Benchmark] public uint UInt32() => default; } -public class EmptyInt64 : RealTimeBenchmarks { [Benchmark] public long Int64() => default; } -public class EmptyUInt64 : RealTimeBenchmarks { [Benchmark] public ulong UInt64() => default; } -public class EmptyIntPtr : RealTimeBenchmarks { [Benchmark] public IntPtr IntPtr() => default; } -public class EmptyUIntPtr : RealTimeBenchmarks { [Benchmark] public UIntPtr UIntPtr() => default; } -public class EmptyVoidPointer : RealTimeBenchmarks { [Benchmark] public unsafe void* VoidPointer() => default; } -public class EmptyClass : RealTimeBenchmarks { [Benchmark] public object Class() => default; } \ No newline at end of file +public class EmptyVoid { [Benchmark] public void Void() { } } +public class EmptyByte { [Benchmark] public byte Byte() => default; } +public class EmptySByte { [Benchmark] public sbyte SByte() => default; } +public class EmptyShort { [Benchmark] public short Short() => default; } +public class EmptyUShort { [Benchmark] public ushort UShort() => default; } +public class EmptyChar { [Benchmark] public char Char() => default; } +public class EmptyInt32 { [Benchmark] public int Int32() => default; } +public class EmptyUInt32 { [Benchmark] public uint UInt32() => default; } +public class EmptyInt64 { [Benchmark] public long Int64() => default; } +public class EmptyUInt64 { [Benchmark] public ulong UInt64() => default; } +public class EmptyIntPtr { [Benchmark] public IntPtr IntPtr() => default; } +public class EmptyUIntPtr { [Benchmark] public UIntPtr UIntPtr() => default; } +public class EmptyVoidPointer { [Benchmark] public unsafe void* VoidPointer() => default; } +public class EmptyClass { [Benchmark] public object Class() => default; } \ No newline at end of file