CoreRT feed and version update (#1606)

* update the integration test

* update CoreRtToolchain to use new CoreRT NuGet feed and compiler version

* update docs

* CoreRtToolchain for TFMs prior to net5.0 should keep using the old CoreRT compiler and old NuGet feed, only .NET5+ should use the new version

* change CoreRT version detection to use .NET Core version detection code (it's well tested)

* re-enable CoreRT(s) test which is possible after the CI got updated to .NET 5 SDK

* update AzureDevOps VM image to windows-2019 which contains VS 2019 and fixes CoreRT build errors

* change the way we install clang 3.9 on Azure Devops Ubuntu

* try suggestion from Michal

* previous fix helped Linux, but broke macOS ;)

Author: adamsitnik

azure-pipelines.Ubuntu.yml

@@ -15,8 +15,8 @@ jobs:
     scriptFileName: ./build.sh
     initialization:
       - bash: |
-          echo "deb https://llvm.org/apt/xenial/ llvm-toolchain-xenial-3.9 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          wget -O - https://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add -
+          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+          sudo apt-add-repository "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main"
           sudo apt-get update
       - bash: |
           sudo apt-get install cmake clang-3.9 libicu55 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev

azure-pipelines.Windows.yml

@@ -11,5 +11,5 @@ jobs:
 - template: build/azure-pipelines.job.template.yml
   parameters:
     name: Windows
-    vmImage: 'vs2017-win2016'
+    vmImage: 'windows-2019'
     scriptFileName: .\build.ps1

docs/articles/configs/toolchains.md

@@ -190,26 +190,26 @@ Example: `dotnet run -c Release -- --coreRun "C:\Projects\corefx\bin\testhost\ne
 
 ## CoreRT
 
-BenchmarkDotNet supports [CoreRT](https://github.com/dotnet/corert)! However, you might want to know how it works to get a better understanding of the results that you get.
+BenchmarkDotNet supports [CoreRT](https://github.com/dotnet/runtimelab/tree/feature/NativeAOT)! However, you might want to know how it works to get a better understanding of the results that you get.
 
 * CoreRT is a flavor of .NET Core. Which means that:
-  *  you have to target .NET Core to be able to build CoreRT benchmarks (`<TargetFramework>netcoreapp2.1</TargetFramework>` in the .csproj file)
-  *  you have to specify the CoreRT runtime in an explicit way, either by using `[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(CoreRtRuntime.$version)`
-  *  to run CoreRT benchmark you run the app as a .NET Core/.NET process (`dotnet run -c Release -f netcoreapp2.1`) and BenchmarkDotNet does all the CoreRT compilation for you. If you want to check what files are generated you need to apply `[KeepBenchmarkFiles]` attribute to the class which defines benchmarks.
+  *  you have to target .NET Core to be able to build CoreRT benchmarks (example: `<TargetFramework>net5.0</TargetFramework>` in the .csproj file)
+  *  you have to specify the CoreRT runtime in an explicit way, either by using `[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(CoreRtRuntime.$version)` or console line arguments `--runtimes corert50`
+  *  to run CoreRT benchmark you run the app as a .NET Core/.NET process (example: `dotnet run -c Release -f net5.01`) and BenchmarkDotNet does all the CoreRT compilation for you. If you want to check what files are generated you need to apply `[KeepBenchmarkFiles]` attribute to the class which defines benchmarks.
 
-By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the CoreRT benchmark according to [this instructions](https://github.com/dotnet/corert/tree/7f902d4d8b1c3280e60f5e06c71951a60da173fb/samples/HelloWorld#add-corert-to-your-project).
+By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the CoreRT benchmark according to [this instructions](https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project).
 
 ```cs
 var config = DefaultConfig.Instance
-    .With(Job.Default.With(CoreRtRuntime.CoreRt21)); // compiles the benchmarks as netcoreapp2.1 and uses the latest CoreRT to build a native app
+    .With(Job.Default.With(CoreRtRuntime.CoreRt50)); // compiles the benchmarks as net5.0 and uses the latest CoreRT to build a native app
 
 BenchmarkSwitcher
     .FromAssembly(typeof(Program).Assembly)
     .Run(args, config);
 ```
 
 ```cs
-[SimpleJob(RuntimeMoniker.CoreRt21)] // compiles the benchmarks as netcoreapp2.1 and uses the latest CoreRT to build a native app
+[SimpleJob(RuntimeMoniker.CoreRt50)] // compiles the benchmarks as net5.0 and uses the latest CoreRT to build a native app
 public class TheTypeWithBenchmarks
 {
    [Benchmark] // the benchmarks go here
@@ -218,15 +218,17 @@ public class TheTypeWithBenchmarks
 
 **Note**: BenchmarkDotNet is going to run `dotnet restore` on the auto-generated project. The first time it does so, it's going to take a **LOT** of time to download all the dependencies (few minutes). Just give it some time and don't press `Ctrl+C` too fast ;)
 
-If you want to benchmark some particular version of CoreRT you have to specify it in an explicit way:
+If you want to benchmark some particular version of CoreRT (or from a different NuGet feed) you have to specify it in an explicit way:
 
 ```cs
 var config = DefaultConfig.Instance
     .With(Job.ShortRun
         .With(CoreRtToolchain.CreateBuilder()
-            .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-26412-02") // the version goes here
+            .UseCoreRtNuGet(
+                microsoftDotNetILCompilerVersion: "6.0.0-*", // the version goes here
+                nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json") // this address might change over time
             .DisplayName("CoreRT NuGet")
-            .TargetFrameworkMoniker("netcoreapp2.1")
+            .TargetFrameworkMoniker("net5.0")
             .ToToolchain()));
 ```
 

src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs

@@ -10,62 +10,60 @@ public class CoreRtRuntime : Runtime
         /// <summary>
         /// CoreRT compiled as netcoreapp2.0
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt20 = new CoreRtRuntime(RuntimeMoniker.CoreRt20, "netcoreapp2.0", "CoreRt 2.0");
+        public static readonly CoreRtRuntime CoreRt20 = new CoreRtRuntime(RuntimeMoniker.CoreRt20, "netcoreapp2.0", "CoreRT 2.0");
         /// <summary>
         /// CoreRT compiled as netcoreapp2.1
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt21 = new CoreRtRuntime(RuntimeMoniker.CoreRt21, "netcoreapp2.1", "CoreRt 2.1");
+        public static readonly CoreRtRuntime CoreRt21 = new CoreRtRuntime(RuntimeMoniker.CoreRt21, "netcoreapp2.1", "CoreRT 2.1");
         /// <summary>
         /// CoreRT compiled as netcoreapp2.2
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt22 = new CoreRtRuntime(RuntimeMoniker.CoreRt22, "netcoreapp2.2", "CoreRt 2.2");
+        public static readonly CoreRtRuntime CoreRt22 = new CoreRtRuntime(RuntimeMoniker.CoreRt22, "netcoreapp2.2", "CoreRT 2.2");
         /// <summary>
         /// CoreRT compiled as netcoreapp3.0
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt30 = new CoreRtRuntime(RuntimeMoniker.CoreRt30, "netcoreapp3.0", "CoreRt 3.0");
+        public static readonly CoreRtRuntime CoreRt30 = new CoreRtRuntime(RuntimeMoniker.CoreRt30, "netcoreapp3.0", "CoreRT 3.0");
         /// <summary>
         /// CoreRT compiled as netcoreapp3.1
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt31 = new CoreRtRuntime(RuntimeMoniker.CoreRt31, "netcoreapp3.1", "CoreRt 3.1");
+        public static readonly CoreRtRuntime CoreRt31 = new CoreRtRuntime(RuntimeMoniker.CoreRt31, "netcoreapp3.1", "CoreRT 3.1");
         /// <summary>
         /// CoreRT compiled as net5.0
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt50 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net5.0", "CoreRt 5.0");
+        public static readonly CoreRtRuntime CoreRt50 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net5.0", "CoreRT 5.0");
         /// <summary>
         /// CoreRT compiled as net6.0
         /// </summary>
-        public static readonly CoreRtRuntime CoreRt60 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net6.0", "CoreRt 6.0");
+        public static readonly CoreRtRuntime CoreRt60 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net6.0", "CoreRT 6.0");
 
         private CoreRtRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName)
             : base(runtimeMoniker, msBuildMoniker, displayName)
         {
         }
 
-        internal static CoreRtRuntime GetCurrentVersion()
+        public static CoreRtRuntime GetCurrentVersion()
         {
             if (!RuntimeInformation.IsNetCore && !RuntimeInformation.IsCoreRT)
             {
                 throw new NotSupportedException("It's impossible to reliably detect the version of CoreRT if the process is not a .NET Core or CoreRT process!");
             }
 
-            var frameworkDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription;
-            string netCoreAppVersion = new string(frameworkDescription.SkipWhile(c => !char.IsDigit(c)).ToArray());
-            string[] versionNumbers = netCoreAppVersion.Split('.');
-            string msBuildMoniker = int.Parse(versionNumbers[0]) >= 5 ? $"net{versionNumbers[0]}.{versionNumbers[1]}" : $"netcoreapp{versionNumbers[0]}.{versionNumbers[1]}";
-            string displayName = $"CoreRT {versionNumbers[0]}.{versionNumbers[1]}";
+            if (!CoreRuntime.TryGetVersion(out var version))
+            {
+                throw new NotSupportedException("Failed to recognize CoreRT version");
+            }
 
-            switch (msBuildMoniker)
+            switch (version)
             {
-                case "netcoreapp2.0": return CoreRt20;
-                case "netcoreapp2.1": return CoreRt21;
-                case "netcoreapp2.2": return CoreRt22;
-                case "netcoreapp3.0": return CoreRt30;
-                case "netcoreapp3.1": return CoreRt31;
-                case "net5.0":
-                case "netcoreapp5.0": return CoreRt50;
-                case "net6.0": return CoreRt60;
-                default: // support future version of CoreRT
-                    return new CoreRtRuntime(RuntimeMoniker.NotRecognized, msBuildMoniker, displayName);
+                case Version v when v.Major == 2 && v.Minor == 0: return CoreRt20;
+                case Version v when v.Major == 2 && v.Minor == 1: return CoreRt21;
+                case Version v when v.Major == 2 && v.Minor == 2: return CoreRt22;
+                case Version v when v.Major == 3 && v.Minor == 0: return CoreRt30;
+                case Version v when v.Major == 3 && v.Minor == 1: return CoreRt31;
+                case Version v when v.Major == 5 && v.Minor == 0: return CoreRt50;
+                case Version v when v.Major == 6 && v.Minor == 0: return CoreRt60;
+                default:
+                    return new CoreRtRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"CoreRT {version.Major}.{version.Minor}");
             }
         }
     }

src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs

@@ -8,31 +8,31 @@ namespace BenchmarkDotNet.Toolchains.CoreRt
     public class CoreRtToolchain : Toolchain
     {
         /// <summary>
-        /// compiled as netcoreapp2.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as netcoreapp2.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core20 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.0").ToToolchain();
         /// <summary>
-        /// compiled as netcoreapp2.1, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as netcoreapp2.1, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core21 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.1").ToToolchain();
         /// <summary>
-        /// compiled as netcoreapp2.2, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as netcoreapp2.2, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core22 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.2").ToToolchain();
         /// <summary>
-        /// compiled as netcoreapp3.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as netcoreapp3.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core30 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.0").ToToolchain();
         /// <summary>
-        /// compiled as netcoreapp3.1, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as netcoreapp3.1, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core31 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.1").ToToolchain();
         /// <summary>
-        /// compiled as net5.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as net5.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core50 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("net5.0").ToToolchain();
         /// <summary>
-        /// compiled as net6.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+        /// compiled as net6.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json
         /// </summary>
         public static readonly IToolchain Core60 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("net6.0").ToToolchain();
 

src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs

@@ -23,12 +23,12 @@ public class CoreRtToolchainBuilder : CustomDotNetCliToolchainBuilder
 
         /// <summary>
         /// creates a CoreRT toolchain targeting NuGet build of CoreRT
-        /// Based on https://github.com/dotnet/corert/blob/7f902d4d8b1c3280e60f5e06c71951a60da173fb/samples/HelloWorld/README.md#add-corert-to-your-project
+        /// Based on https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project
         /// </summary>
-        /// <param name="microsoftDotNetILCompilerVersion">the version of Microsoft.DotNet.ILCompiler which should be used. The default is: "1.0.0-alpha-*"</param>
-        /// <param name="nuGetFeedUrl">url to NuGet CoreRT feed, The default is: "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"</param>
+        /// <param name="microsoftDotNetILCompilerVersion">the version of Microsoft.DotNet.ILCompiler which should be used. The default is: "6.0.0-*"</param>
+        /// <param name="nuGetFeedUrl">url to NuGet CoreRT feed, The default is: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"</param>
         [PublicAPI]
-        public CoreRtToolchainBuilder UseCoreRtNuGet(string microsoftDotNetILCompilerVersion = "1.0.0-alpha-*", string nuGetFeedUrl = "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json")
+        public CoreRtToolchainBuilder UseCoreRtNuGet(string microsoftDotNetILCompilerVersion = "6.0.0-*", string nuGetFeedUrl = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json")
         {
             coreRtVersion = microsoftDotNetILCompilerVersion ?? throw new ArgumentNullException(nameof(microsoftDotNetILCompilerVersion));
 

src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs

@@ -136,6 +136,7 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif
     <RootAllApplicationAssemblies>{rootAllApplicationAssemblies}</RootAllApplicationAssemblies>
     <IlcGenerateCompleteTypeMetadata>{ilcGenerateCompleteTypeMetadata}</IlcGenerateCompleteTypeMetadata>
     <IlcGenerateStackTraceData>{ilcGenerateStackTraceData}</IlcGenerateStackTraceData>
+    <EnsureNETCoreAppRuntime>false</EnsureNETCoreAppRuntime> <!-- workaround for 'This runtime may not be supported by.NET Core.' error -->
   </PropertyGroup>
   {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)}
   <ItemGroup>
@@ -148,6 +149,9 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif
   <ItemGroup>
     <RdXmlFile Include=""rd.xml"" />
   </ItemGroup>
+  <ItemGroup Condition="" '$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' "">
+    <LinkerArg Include=""-lanl"" />
+  </ItemGroup>
 </Project>";
 
         private string GenerateProjectForLocalBuild(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) => $@"
@@ -182,6 +186,9 @@ private string GenerateProjectForLocalBuild(BuildPartition buildPartition, Artif
   <ItemGroup>
     <RdXmlFile Include=""rd.xml"" />
   </ItemGroup>
+  <ItemGroup Condition="" '$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' "">
+    <LinkerArg Include=""-lanl"" />
+  </ItemGroup>
 </Project>";
 
         /// <summary>

tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalCoreRtToolchainTests.cs

@@ -29,7 +29,7 @@ public void CanBenchmarkLocalCoreRtUsingRyuJit()
         {
             var config = ManualConfig.CreateEmpty()
                 .AddJob(Job.Dry
-                    .WithRuntime(CoreRtRuntime.CoreRt21)
+                    .WithRuntime(CoreRtRuntime.CoreRt50)
                     .WithToolchain(
                         CoreRtToolchain.CreateBuilder()
                             .UseCoreRtLocal(IlcPath)
@@ -43,7 +43,7 @@ public void CanBenchmarkLocalCoreRtUsingCppCodeGen()
         {
             var config = ManualConfig.CreateEmpty()
                 .AddJob(Job.Dry
-                    .WithRuntime(CoreRtRuntime.CoreRt21)
+                    .WithRuntime(CoreRtRuntime.CoreRt50)
                     .WithToolchain(
                         CoreRtToolchain.CreateBuilder()
                             .UseCoreRtLocal(IlcPath)

tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs

@@ -26,9 +26,9 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled()
 
             var config = ManualConfig.CreateEmpty()
                 .AddJob(Job.Dry
-                    .WithRuntime(CoreRtRuntime.CoreRt21)
+                    .WithRuntime(CoreRtRuntime.CoreRt50)
                     .WithToolchain(CoreRtToolchain.CreateBuilder()
-                        .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-27408-02") // we test against specific version to keep this test stable
+                        .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-alpha.1.20602.1") // we test against specific version to keep this test stable
                         .Timeout(timeout)
                         .ToToolchain()));