.Net SDK Validation (#2523)

* Adding DotNetSDK Validation

* Added support for Wasm and NativeAot

* Added missing case statements

* Code cleanup

* Separated providers for .Net and .Net Framework. Added validation to MonoAotLLVMToolChain and CsProjClassicNetToolchain

* Put code into static utility class. Updated toolchains

* Updated validator and toolchains

* Updated toolchains and framework validation

Author: MattFromRVA

src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs

@@ -66,7 +66,7 @@ public IEnumerable<ValidationError> Validate(BenchmarkCase benchmark, IResolver
                     $"Provided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder.",
                     benchmark);
             }
-            else if (Toolchain.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError))
+            else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError))
             {
                 yield return invalidCliError;
             }

src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs

@@ -49,11 +49,17 @@ public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCas
                 yield return new ValidationError(true,
                     $"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed",
                     benchmarkCase);
+                yield break;
             }
-            else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError))
+            else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError))
             {
                 yield return invalidCliError;
             }
+
+            foreach (var validationError in DotNetSdkValidator.ValidateFrameworkSdks(benchmarkCase))
+            {
+                yield return validationError;
+            }
         }
     }
 }

src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs

@@ -49,11 +49,6 @@ public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCas
                 yield return validationError;
             }
 
-            if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError))
-            {
-                yield return invalidCliError;
-            }
-
             if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit)
             {
                 yield return new ValidationError(true,
@@ -80,6 +75,11 @@ public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCas
                     $"Currently CsProjCoreToolchain does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead.",
                     benchmarkCase);
             }
+
+            foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase))
+            {
+                yield return validationError;
+            }
         }
 
         public override bool Equals(object obj) => obj is CsProjCoreToolchain typed && Equals(typed);

src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs

@@ -1,24 +1,45 @@
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Validators;
+using System.Collections.Generic;
 
 namespace BenchmarkDotNet.Toolchains.MonoAotLLVM
 {
     public class MonoAotLLVMToolChain : Toolchain
     {
-        public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor)
+        private readonly string _customDotNetCliPath;
+
+        public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath)
             : base(name, generator, builder, executor)
         {
+            _customDotNetCliPath = customDotNetCliPath;
         }
 
         public static IToolchain From(NetCoreAppSettings netCoreAppSettings)
-            => new MonoAotLLVMToolChain(netCoreAppSettings.Name,
-                    new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker,
-                        netCoreAppSettings.CustomDotNetCliPath,
-                        netCoreAppSettings.PackagesPath,
-                        netCoreAppSettings.CustomRuntimePack,
-                        netCoreAppSettings.AOTCompilerPath,
-                        netCoreAppSettings.AOTCompilerMode),
-                    new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker,
-                        netCoreAppSettings.CustomDotNetCliPath),
-                    new Executor());
+        => new MonoAotLLVMToolChain(netCoreAppSettings.Name,
+                new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker,
+                    netCoreAppSettings.CustomDotNetCliPath,
+                    netCoreAppSettings.PackagesPath,
+                    netCoreAppSettings.CustomRuntimePack,
+                    netCoreAppSettings.AOTCompilerPath,
+                    netCoreAppSettings.AOTCompilerMode),
+                new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker,
+                    netCoreAppSettings.CustomDotNetCliPath),
+                new Executor(),
+                netCoreAppSettings.CustomDotNetCliPath);
+
+        public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCase, IResolver resolver)
+        {
+            foreach (var validationError in base.Validate(benchmarkCase, resolver))
+            {
+                yield return validationError;
+            }
+
+            foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(_customDotNetCliPath, benchmarkCase))
+            {
+                yield return validationError;
+            }
+        }
     }
 }

src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs

@@ -32,9 +32,10 @@ public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCas
                     $"{nameof(WasmToolchain)} is supported only on Unix, benchmark '{benchmarkCase.DisplayInfo}' might not work correctly",
                     benchmarkCase);
             }
-            else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError))
+
+            foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase))
             {
-                yield return invalidCliError;
+                yield return validationError;
             }
         }
 

src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs

@@ -1,5 +1,8 @@
 using System.Collections.Generic;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Validators;
 
 namespace BenchmarkDotNet.Toolchains.NativeAot
 {
@@ -60,5 +63,18 @@ internal NativeAotToolchain(string displayName,
         public static NativeAotToolchainBuilder CreateBuilder() => NativeAotToolchainBuilder.Create();
 
         public static string GetExtraArguments(string runtimeIdentifier) => $"-r {runtimeIdentifier}";
+
+        public override IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCase, IResolver resolver)
+        {
+            foreach (var error in base.Validate(benchmarkCase, resolver))
+            {
+                yield return error;
+            }
+
+            foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase))
+            {
+                yield return validationError;
+            }
+        }
     }
 }

src/BenchmarkDotNet/Toolchains/Toolchain.cs

@@ -57,31 +57,6 @@ public virtual IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCase
             }
         }
 
-        internal static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase benchmarkCase, out ValidationError? validationError)
-        {
-            validationError = null;
-
-            if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled())
-            {
-                validationError = new ValidationError(true,
-                    $"BenchmarkDotNet requires dotnet SDK to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed",
-                    benchmarkCase);
-
-                return true;
-            }
-
-            if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath))
-            {
-                validationError = new ValidationError(true,
-                    $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed",
-                    benchmarkCase);
-
-                return true;
-            }
-
-            return false;
-        }
-
         public override string ToString() => Name;
     }
-}
+}