Add NativeAOT (and CoreRT) support for micro benchmarks (#2328)

* prefer AppContext.BaseDirectory over typeof(X).Assembly.Location

* GetReferencedAssemblies() is not supported on AOT

* Arrays with non-zero lower bounds are not supported

* BinaryFormatter serialization and deserialization are disabled within this application.

* Dynamic code generation is not supported on this platform.

* disable AddRemoveFromDifferentThreads as it hangs

* disable more serialization benchmarks

* disable Activator.CreateInstance(_assemblyName, _typeName) benchmark as it's not supported

* add NativeAOT and CoreRT support to the python scripts

* bump BDN version

* address code review feedback, don't use the CoreRT name anymore

* add NativeAOT CI leg

* increase build timeout

azure-pipelines.yml

@@ -2,6 +2,8 @@ resources:
   containers:
     - container: ubuntu_x64_build_container
       image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20210311173856-047508b
+    - container: centos_x64_build_container
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-20220107135107-9b5bbc2
 
 # CI Trigger on main branch
 trigger:
@@ -233,6 +235,23 @@ jobs:
       channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks
         - main
 
+  # Ubuntu 1804 x64 NativeAOT micro benchmarks
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: centos
+      osVersion: 7
+      kind: micro
+      architecture: x64
+      pool: 
+        vmImage: ubuntu-latest
+      machinePool: Open
+      queue: centos.7.amd64.open.rt
+      container: centos_x64_build_container
+      csproj: src/benchmarks/micro/MicroBenchmarks.csproj
+      runCategories: 'runtime libraries'
+      channels:
+        - nativeaot7.0
+
   # Ubuntu 1804 x64 ML.NET benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:

eng/performance/benchmark_jobs.yml

@@ -58,6 +58,12 @@ jobs:
           - ${{ if or(eq(parameters.osVersion, '1604'), eq(parameters.architecture, 'arm64'))}}:
             - name: ArtifactsDirectory
               value: '$HELIX_WORKITEM_PAYLOAD/artifacts/BenchmarkDotNet.Artifacts'
+          - ${{ if eq(parameters.osName, 'ubuntu')}}:
+            - name: InstallPrerequisites
+              value: 'sudo apt-get -y install python3-venv'
+          - ${{ if eq(parameters.osName, 'centos')}}:
+            - name: InstallPrerequisites
+              value: 'sudo yum -y install clang zlib-devel krb5-libs krb5-devel'
         - ${{ if eq(variables['System.TeamProject'], 'public') }}:
           # for public runs, we do not want to upload perflab data
           - name: PerfLabArguments
@@ -70,7 +76,7 @@ jobs:
           - name: HelixApiAccessToken
             value: ''
           - name: HelixPreCommand
-            value: '$(PreservePythonPath);$(AdditionalHelixPreCommands)'
+            value: '$(PreservePythonPath);$(AdditionalHelixPreCommands);$(InstallPrerequisites);'
           - name: HelixSourcePrefix
             value: "pr"
         - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -89,7 +95,7 @@ jobs:
               value: '$(PreservePythonPath);py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)";$(AdditionalHelixPreCommands)'
           - ${{ if ne(parameters.osName, 'windows') }}:
             - name: HelixPreCommand
-              value: '$(PreservePythonPath);$(SetAllowOpenSsl102);sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;. $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0 --force-reinstall;pip3 install azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)";$(AdditionalHelixPreCommands)'
+              value: '$(PreservePythonPath);$(SetAllowOpenSsl102);$(InstallPrerequisites);python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;. $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0 --force-reinstall;pip3 install azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)";$(AdditionalHelixPreCommands)'
           - group: DotNet-HelixApi-Access
           # perflab upload tokens still exist in this variable group
           - group: dotnet-benchview

scripts/channel_map.py

@@ -37,10 +37,20 @@ class ChannelMap():
             'branch': '7.0.1xx',
             'quality': 'daily'
         },
+        'nativeaot7.0': {
+            'tfm': 'nativeaot7.0',
+            'branch': '7.0.1xx',
+            'quality': 'daily'
+        },
         'master': {
             'tfm': 'net6.0',
             'branch': 'master'
         },
+        'nativeaot6.0': {
+            'tfm': 'nativeaot6.0',
+            'branch': '6.0.1xx',
+            'quality': 'daily'
+        },
         '5.0':{
             'tfm': 'net5.0',
             'branch': '5.0.4xx',

scripts/dotnet.py

@@ -78,24 +78,29 @@ def __call__(self, parser, namespace, values, option_string=None):
     def get_target_framework_moniker(framework: str) -> str:
         '''
         Translates framework name to target framework moniker (TFM)
-        To run CoreRT benchmarks we need to run the host BDN process as latest
-        .NET Core the host process will build and run CoreRT benchmarks
+        To run NativeAOT benchmarks we need to run the host BDN process as latest
+        .NET the host process will build and run AOT benchmarks
         '''
-        return ChannelMap.get_target_framework_moniker("main") if framework == 'corert' else framework
+        if framework == 'nativeaot6.0':
+            return 'net6.0'
+        if framework == 'nativeaot7.0':
+            return 'net7.0'
+        else:
+            return framework
 
     @staticmethod
     def get_target_framework_monikers(frameworks: list) -> list:
         '''
         Translates framework names to target framework monikers (TFM)
-        Required to run CoreRT benchmarks where the host process must be .NET
-        Core, not CoreRT.
+        Required to run AOT benchmarks where the host process must be .NET
+        , not NativeAOT.
         '''
         monikers = [
             FrameworkAction.get_target_framework_moniker(framework)
             for framework in frameworks
         ]
 
-        # ['net5.0', 'corert'] should become ['net5.0']
+        # ['net6.0', 'nativeaot6.0'] should become ['net6.0']
         return list(set(monikers))
 
 class VersionsAction(Action):
@@ -136,7 +141,7 @@ class CompilationAction(Action):
     For PRs it is recommended to kick off a Tiered run, and being able to
     manually kick-off NoTiering and FullyJittedNoTiering modes when needed.
     '''
-    # TODO: Would 'Default' make sense for .NET Framework / CoreRT / Mono?
+    # TODO: Would 'Default' make sense for .NET Framework / NativeAOT / Mono?
     # TODO: Should only be required for benchmark execution under certain tools
 
     TIERED = 'Tiered'

scripts/micro_benchmarks.py

@@ -255,8 +255,10 @@ def __get_benchmarkdotnet_arguments(framework: str, args: tuple) -> list:
     # if we don't it's gonna restore to default global folder
     run_args += ['--packages', get_packages_directory()]
 
-    # Required for CoreRT where:
+    # Required for WASM and NativeAOT where:
     #   host process framework != benchmark process framework
+    if framework.startswith("nativeaot"):
+        run_args += ['--runtimes', framework]
     if args.wasm:
         if framework == "net5.0" or framework == "net6.0":
             run_args += ['--runtimes', 'wasm']

src/benchmarks/micro/Program.cs

@@ -45,7 +45,7 @@ static int Main(string[] args)
                 .FromAssembly(typeof(Program).Assembly)
                 .Run(argsList.ToArray(), 
                     RecommendedConfig.Create(
-                        artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "BenchmarkDotNet.Artifacts")), 
+                        artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), 
                         mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty),
                         partitionCount: partitionCount,
                         partitionIndex: partitionIndex,

src/benchmarks/micro/Serializers/Binary_FromStream.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks.Serializers.Helpers;
 
 namespace MicroBenchmarks.Serializers
@@ -16,6 +17,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     [BenchmarkCategory(Categories.NoWASM)]
+    [AotFilter("Dynamic code generation is not supported.")]
     public class Binary_FromStream<T>
     {
         private T value;

src/benchmarks/micro/Serializers/Binary_ToStream.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks.Serializers.Helpers;
 
 namespace MicroBenchmarks.Serializers
@@ -16,6 +17,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     [BenchmarkCategory(Categories.NoWASM)]
+    [AotFilter("Dynamic code generation is not supported.")]
     public class Binary_ToStream<T>
     {
         private readonly T value;

src/benchmarks/micro/Serializers/Json_FromStream.cs

@@ -6,6 +6,7 @@
 using System.Runtime.Serialization.Json;
 using System.Text;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -15,6 +16,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     [BenchmarkCategory(Categories.NoAOT)]
+    [AotFilter("Dynamic code generation is not supported.")]
     public class Json_FromStream<T>
     {
         private T value;

src/benchmarks/micro/Serializers/Json_FromString.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -12,6 +13,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     [BenchmarkCategory(Categories.NoAOT)]
+    [AotFilter("Dynamic code generation is not supported.")]
     public class Json_FromString<T>
     {
         private string serialized;

src/benchmarks/micro/Serializers/Json_ToStream.cs

@@ -6,6 +6,7 @@
 using System.Runtime.Serialization.Json;
 using System.Text;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -15,6 +16,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     [BenchmarkCategory(Categories.NoAOT)]
+    [AotFilter("Dynamic code generation is not supported.")]
     public class Json_ToStream<T>
     {
         private T value;

src/benchmarks/micro/Serializers/Json_ToString.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -12,6 +13,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     [BenchmarkCategory(Categories.NoAOT)]
+    [AotFilter("Dynamic code generation is not supported.")]
     public class Json_ToString<T>
     {
         private T value;

src/benchmarks/micro/Serializers/Xml_FromStream.cs

@@ -7,6 +7,7 @@
 using System.Xml;
 using System.Xml.Serialization;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -18,6 +19,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(XmlElement))]
     [GenericTypeArguments(typeof(SimpleStructWithProperties))]
     [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class Xml_FromStream<T>
     {
         private T value;

src/benchmarks/micro/Serializers/Xml_ToStream.cs

@@ -7,6 +7,7 @@
 using System.Xml;
 using System.Xml.Serialization;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 
 namespace MicroBenchmarks.Serializers
 {
@@ -18,6 +19,7 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(XmlElement))]
     [GenericTypeArguments(typeof(SimpleStructWithProperties))]
     [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class Xml_ToStream<T>
     {
         private T value;

src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs

@@ -6,6 +6,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks;
 
 namespace System.Collections.Concurrent
@@ -15,6 +16,7 @@ namespace System.Collections.Concurrent
     [GenericTypeArguments(typeof(string))] // reference type
     [MinWarmupCount(6, forceAutoWarmup: true)]
     [MaxWarmupCount(10, forceAutoWarmup: true)]
+    [AotFilter("It hangs. https://github.com/dotnet/runtime/issues/66987")]
     public class AddRemoveFromDifferentThreads<T>
     {
         const int NumThreads = 2;

src/benchmarks/micro/libraries/System.IO.Compression/CompressedFile.cs

@@ -34,7 +34,7 @@ public CompressedFile(string fileName, CompressionLevel compressionLevel, Func<S
 
         internal static string GetFilePath(string fileName) 
             => Path.Combine(
-                Path.GetDirectoryName(typeof(CompressedFile).Assembly.Location), 
+                AppContext.BaseDirectory,
                 "libraries", "System.IO.Compression", "TestData",
                 fileName);
     }

src/benchmarks/micro/libraries/System.Net.Http/Configuration.Certificates.cs

@@ -27,7 +27,7 @@ private static X509Certificate2 GetCertificate(string certificateFileName)
                 => new X509Certificate2(
                     File.ReadAllBytes(
                         Path.Combine(
-                            Path.GetDirectoryName(typeof(Certificates).Assembly.Location), 
+                            AppContext.BaseDirectory, 
                             "libraries", 
                             "System.Net.Http", 
                             certificateFileName)),

src/benchmarks/micro/libraries/System.Runtime.Serialization.Formatters/Perf.BinaryFormatter.cs

@@ -7,11 +7,13 @@
 using System.Linq;
 using System.Runtime.Serialization.Formatters.Binary;
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks;
 
 namespace System.Runtime.Serialization.Formatters.Tests
 {
     [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)]
+    [AotFilter("Disabled by default.")]
     public class Perf_BinaryFormatter
     {
         private readonly BinaryFormatter _formatter = new BinaryFormatter();

src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs

@@ -3,13 +3,15 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks;
 using MicroBenchmarks.Serializers;
 
 namespace System.Text.Json.Serialization.Tests
 {
     [BenchmarkCategory(Categories.Libraries, Categories.JSON)]
     [GenericTypeArguments(typeof(SimpleStructWithProperties))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class ColdStartSerialization<T>
     {
         // Measures cold start performance for the serializer

src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks;
 using MicroBenchmarks.Serializers;
 using System.Collections;
@@ -28,6 +29,7 @@ namespace System.Text.Json.Serialization.Tests
     [GenericTypeArguments(typeof(LargeStructWithProperties))]
     [GenericTypeArguments(typeof(DateTimeOffset?))]
     [GenericTypeArguments(typeof(int))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class ReadJson<T>
     {
         private string _serialized;

src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadMissingAndCaseInsensitive.cs

@@ -3,13 +3,15 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks;
 using MicroBenchmarks.Serializers;
 
 namespace System.Text.Json.Serialization.Tests
 {
     [BenchmarkCategory(Categories.Libraries, Categories.JSON)]
     [GenericTypeArguments(typeof(Location))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class ReadMissingAndCaseInsensitive<T>
     {
         private string _serialized;

src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadPreservedReferences.cs

@@ -15,6 +15,7 @@ namespace System.Text.Json.Serialization.Tests
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
     [GenericTypeArguments(typeof(SimpleListOfInt))]
     [GenericTypeArguments(typeof(SimpleStructWithProperties))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class ReadPreservedReferences<T>
     {
         [Params(false, true)]

src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes.Filters;
 using MicroBenchmarks;
 using MicroBenchmarks.Serializers;
 using System.Collections;
@@ -29,6 +30,7 @@ namespace System.Text.Json.Serialization.Tests
     [GenericTypeArguments(typeof(LargeStructWithProperties))]
     [GenericTypeArguments(typeof(DateTimeOffset?))]
     [GenericTypeArguments(typeof(int))]
+    [AotFilter("Currently not supported due to missing metadata.")]
     public class WriteJson<T>
     {
         private T _value;