Add Roslyn analyzers to detect incorrect usage of BenchmarkDotNet

.github/workflows/run-tests.yaml

@@ -45,6 +45,12 @@ jobs:
       - name: Run task 'build'
         shell: cmd
         run: ./build.cmd build
+      - name: Run task 'unit-tests'
+        shell: cmd
+        run: ./build.cmd unit-tests -e
+      - name: Run task 'analyzer-tests'
+        shell: cmd
+        run: ./build.cmd analyzer-tests -e
       - name: Run task 'in-tests-full'
         shell: cmd
         run: ./build.cmd in-tests-full -e
@@ -79,6 +85,8 @@ jobs:
       # Build and Test
       - name: Run task 'build'
         run: ./build.cmd build
+      - name: Run task 'analyzer-tests'
+        run: ./build.cmd analyzer-tests -e
       - name: Run task 'unit-tests'
         run: ./build.cmd unit-tests -e
       - name: Run task 'in-tests-core'
@@ -116,6 +124,8 @@ jobs:
       # Build and Test
       - name: Run task 'build'
         run: ./build.cmd build
+      - name: Run task 'analyzer-tests'
+        run: ./build.cmd analyzer-tests -e
       - name: Run task 'unit-tests'
         run: ./build.cmd unit-tests -e
       - name: Run task 'in-tests-core'

.gitignore

@@ -54,6 +54,9 @@ src/BenchmarkDotNet/Disassemblers/BenchmarkDotNet.Disassembler.*.nupkg
 # Visual Studio 2015 cache/options directory
 .vs/
 
+# VSCode directory
+.vscode/
+
 # Cake
 tools/**
 .dotnet

BenchmarkDotNet.sln

@@ -59,6 +59,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.P
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Analyzers", "src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj", "{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Analyzers.Tests", "tests\BenchmarkDotNet.Analyzers.Tests\BenchmarkDotNet.Analyzers.Tests.csproj", "{7DE89F16-2160-42E3-004E-1F5064732121}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -161,6 +165,14 @@ Global
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7DE89F16-2160-42E3-004E-1F5064732121}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -190,6 +202,8 @@ Global
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
+		{AA4DDCA0-C1D8-ADA8-69FE-2F67C4CA96B1} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
+		{7DE89F16-2160-42E3-004E-1F5064732121} = {14195214-591A-45B7-851A-19D3BA2413F9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

NuGet.Config

@@ -1,18 +1,21 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <solution>
-    <add key="disableSourceControlIntegration" value="true" />
-  </solution>
-  <packageSources>
-    <!--To inherit the global NuGet package sources remove the <clear/> line below -->
-    <clear />
-
-    <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
-    <!-- reuquired to run Mono AOT benchmarks -->
-    <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
-    <add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
-    <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
-    <add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
-    <add key="dotnet10" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json" />
-  </packageSources>
-</configuration>
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+  <packageSources>
+    <!--To inherit the global NuGet package sources remove the <clear/> line below -->
+    <clear />
+
+    <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <!-- required to run Mono AOT benchmarks -->
+    <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
+    <add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
+    <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
+    <add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
+    <add key="dotnet10" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json" />
+
+    <!-- required for Roslyn analyzers -->
+	<add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
+  </packageSources>
+</configuration>

build/BenchmarkDotNet.Build/Program.cs

@@ -98,6 +98,28 @@ public HelpInfo GetHelp()
     }
 }
 
+[TaskName(Name)]
+[TaskDescription("Run analyzer tests")]
+[IsDependentOn(typeof(BuildTask))]
+public class AnalyzerTestsTask : FrostingTask<BuildContext>, IHelpProvider
+{
+    private const string Name = "analyzer-tests";
+    public override void Run(BuildContext context) => context.UnitTestRunner.RunAnalyzerTests();
+
+    public HelpInfo GetHelp()
+    {
+        return new HelpInfo
+        {
+            Examples =
+            [
+                new Example(Name)
+                    .WithArgument(KnownOptions.Exclusive)
+                    .WithArgument(KnownOptions.Verbosity, "Diagnostic")
+            ]
+        };
+    }
+}
+
 [TaskName(Name)]
 [TaskDescription("Run integration tests using .NET Framework 4.6.2+ (slow)")]
 [IsDependentOn(typeof(BuildTask))]
@@ -123,8 +145,9 @@ public class InTestsCoreTask : FrostingTask<BuildContext>, IHelpProvider
 }
 
 [TaskName(Name)]
-[TaskDescription("Run all unit and integration tests (slow)")]
+[TaskDescription("Run all unit, analyzer, and integration tests (slow)")]
 [IsDependentOn(typeof(UnitTestsTask))]
+[IsDependentOn(typeof(AnalyzerTestsTask))]
 [IsDependentOn(typeof(InTestsFullTask))]
 [IsDependentOn(typeof(InTestsCoreTask))]
 public class AllTestsTask : FrostingTask<BuildContext>, IHelpProvider

build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs

@@ -20,6 +20,11 @@ public class UnitTestRunner(BuildContext context)
         .Combine("BenchmarkDotNet.Exporters.Plotting.Tests")
         .CombineWithFilePath("BenchmarkDotNet.Exporters.Plotting.Tests.csproj");
 
+    private FilePath AnalyzerTestsProjectFile { get; } = context.RootDirectory
+        .Combine("tests")
+        .Combine("BenchmarkDotNet.Analyzers.Tests")
+        .CombineWithFilePath("BenchmarkDotNet.Analyzers.Tests.csproj");
+
     private FilePath IntegrationTestsProjectFile { get; } = context.RootDirectory
         .Combine("tests")
         .Combine("BenchmarkDotNet.IntegrationTests")
@@ -70,5 +75,12 @@ public void RunUnitTests()
             RunUnitTests(targetFramework);
     }
 
+    public void RunAnalyzerTests()
+    {
+        string[] targetFrameworks = context.IsRunningOnWindows() ? ["net462", "net8.0"] : ["net8.0"];
+        foreach (var targetFramework in targetFrameworks)
+            RunTests(AnalyzerTestsProjectFile, "analyzer", targetFramework);
+    }
+
     public void RunInTests(string tfm) => RunTests(IntegrationTestsProjectFile, "integration", tfm);
 }

build/common.props

@@ -21,10 +21,11 @@
     <CheckEolTargetFramework>false</CheckEolTargetFramework>
     <CodeAnalysisRuleSet>$(MSBuildThisFileDirectory)CodingStyle.ruleset</CodeAnalysisRuleSet>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-
+    <WarningsNotAsErrors>NU1900</WarningsNotAsErrors>
     <Nullable>annotations</Nullable>
     <!-- Suppress warning for nuget package used in old (unsupported) tfm. -->
     <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
+    <NoWarn>CS9057</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>

samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj

@@ -40,4 +40,8 @@
     <ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />
     <ProjectReference Include="..\..\src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj" />
   </ItemGroup>
+  <ItemGroup>
+    <!-- Enables analyzers for this project (this is required since ProjectReference is not transitive). -->
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj" ReferenceOutputAssembly="false" OutputItemType="Analyzer"/>
+  </ItemGroup>
 </Project>