Merge pull request #343 from microsoft/manodasanw/benchmarks

Adding first set of benchmarks and infrastructure for it

Benchmarks/Benchmarks.csproj

@@ -0,0 +1,55 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <OutputType>Exe</OutputType>
+ <Platforms>x64;x86</Platforms>
+ <TargetFrameworks>netcoreapp2.0;net5.0;netcoreapp3.1</TargetFrameworks>
+ <UseWinmd>false</UseWinmd>
+ <BenchmarkWinmdSupport Condition="'$(BenchmarkWinmdSupport)' == ''">false</BenchmarkWinmdSupport>
+ <!-- Controls whether to build using WinMDs or the projection. -->
+ <UseWinmd Condition="'$(TargetFramework)' == 'netcoreapp3.1' And $(BenchmarkWinmdSupport) == true">true</UseWinmd>
+ <ApplicationManifest Condition="$(UseWinmd) == true">Benchmarks.manifest</ApplicationManifest>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
+ <PackageReference Include="Microsoft.Windows.SDK.Contracts" Version="10.0.18362.2005" Condition="$(UseWinmd) == true"/>
+ <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.5.3" Condition="$(UseWinmd) == false"/>
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\Projections\Windows\Windows.csproj" Condition="$(UseWinmd) == false" />
+ <ProjectReference Include="..\Projections\Benchmark\Benchmark.csproj" Condition="$(UseWinmd) == false" />
+ <ProjectReference Include="..\TestWinRT\BenchmarkComponent\BenchmarkComponent.vcxproj" Condition="$(UseWinmd) == true" />
+
+ <!-- The Benchmark / BenchmarkCompnent reference doesn't seem to bring over the dll when building with dotnet cli, so including both dll and winmd as includes. -->
+ <None Include="$(MSBuildThisFileDirectory)..\_build\$(Platform)\$(Configuration)\BenchmarkComponent\bin\BenchmarkComponent\BenchmarkComponent.dll">
+ <Link>BenchmarkComponent.dll</Link>
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+ <Visible>True</Visible>
+ </None>
+
+ <Reference Include="BenchmarkComponent.winmd" Condition="$(UseWinmd) == true">
+ <HintPath>$(MSBuildThisFileDirectory)..\_build\$(Platform)\$(Configuration)\BenchmarkComponent\bin\BenchmarkComponent\BenchmarkComponent.winmd</HintPath>
+ <IsWinMDFile>true</IsWinMDFile>
+ </Reference>
+
+ <!-- When building for NetCoreApp 3.1, we test the WinMD scenario via reg free winrt, so it needs to be in the bin folder for the generated project to reference it. -->
+ <None Include="Benchmarks.manifest" Condition="$(UseWinmd) == true">
+ <Link>Benchmarks.manifest</Link>
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+ <Visible>True</Visible>
+ </None>
+
+ </ItemGroup>
+
+ <Target Name="FilterProjection" BeforeTargets="CoreCompile" Condition="$(UseWinmd) == false">
+ <ItemGroup>
+ <!--Remove references to projection source winmds to prevent compile conflict warnings-->
+ <ReferencePathWithRefAssemblies Remove="@(ReferencePathWithRefAssemblies)" Condition="%(ReferencePathWithRefAssemblies.Filename) == 'BenchmarkComponent'" />
+ <!--Also remove ReferencePath winmds to prevent error NETSDK1130 false positive-->
+ <ReferencePath Remove="@(ReferencePath)" Condition="%(ReferencePath.Filename) == 'BenchmarkComponent'" />
+ </ItemGroup>
+ </Target>
+
+</Project>

Benchmarks/Benchmarks.manifest

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+ <assemblyIdentity version="1.0.0.0" name="Benchmarks.app"/>
+
+ <file name="BenchmarkComponent.dll">
+ <activatableClass
+ name="BenchmarkComponent.ClassWithMultipleInterfaces"
+ threadingModel="both"
+ xmlns="urn:schemas-microsoft-com:winrt.v1" />
+ </file>
+
+</assembly>

Benchmarks/Program.cs

@@ -0,0 +1,95 @@
+using BenchmarkDotNet.Running;
+using System;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Characteristics;
+using System.IO;
+
+namespace Benchmarks
+{
+ public class Program
+ {
+ static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new CustomConfig().Config);
+
+ private class CustomConfig : Attribute, IConfigSource
+ {
+ public IConfig Config { get; } = DefaultConfig.Instance;
+
+ public CustomConfig()
+ {
+ // Test CsWinRT projection
+ var job = Job.Default
+ .WithPlatform(BenchmarkDotNet.Environments.Platform.X64)
+ .WithArguments(
+ new Argument[] {
+ new MsBuildArgument("/p:platform=x64")
+ }
+ ).AsDefault();
+
+ // Test WinMD support
+#if NETCOREAPP3_1
+ // BenchmarkDotNet will rebuild the project with a project reference to this project when this project's output exe is ran. It
+ // will be ran from the same folder as where we have the application manifest binplaced which we want to embed in the new exe.
+ string manifestFile = Path.Combine(
+ Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location),
+ "Benchmarks.manifest");
+
+ var winmdJob = Job.Default
+ .WithPlatform(BenchmarkDotNet.Environments.Platform.X64)
+ .WithToolchain(new NetCore3ToolChainWithNativeExecution())
+ .WithArguments(
+ new Argument[] {
+ new MsBuildArgument("/p:platform=x64"),
+ new MsBuildArgument("/p:ApplicationManifest=" + manifestFile),
+ new MsBuildArgument("/p:BenchmarkWinmdSupport=true")
+ }
+ )
+ .WithId("WinMD NetCoreApp31");
+
+ // Optimizer needs to be diabled as it errors on WinMDs
+ Config = Config.WithOption(ConfigOptions.DisableOptimizationsValidator, true)
+ .AddJob(winmdJob);
+#else
+ Config = Config.AddJob(job);
+#endif
+ }
+ }
+
+ // Custom tool chain for building the benchmark with WinMDs as we need to execute the
+ // exe version of the benchmark rather than the dll version which runs under dotnet cli.
+ // This is because we need to be able to embed a side by side manifest for reg free winrt
+ // and we need COM to be able to find the WinMDs.
+ private class NetCore3ToolChainWithNativeExecution : Toolchain
+ {
+ public NetCore3ToolChainWithNativeExecution()
+ : base("netcoreapp3.1-native",
+ new CsProjGeneratorWithNativeExe(NetCoreAppSettings.NetCoreApp31),
+ CsProjCoreToolchain.NetCoreApp31.Builder,
+ new Executor())
+ {
+ }
+
+ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IResolver resolver)
+ {
+ return CsProjCoreToolchain.NetCoreApp31.IsSupported(benchmarkCase, logger, resolver);
+ }
+ }
+
+ private class CsProjGeneratorWithNativeExe : CsProjGenerator
+ {
+ public CsProjGeneratorWithNativeExe(NetCoreAppSettings settings)
+ :base(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.RuntimeFrameworkVersion)
+ {
+ }
+
+ protected override string GetExecutableExtension()
+ {
+ return ".exe";
+ }
+ }
+ }
+}

Benchmarks/QueryInterface.cs

@@ -0,0 +1,79 @@
+using BenchmarkComponent;
+using BenchmarkDotNet.Attributes;
+using Windows.ApplicationModel.Chat;
+
+namespace Benchmarks
+{
+ [MemoryDiagnoser]
+ public class QueryInterfacePerf
+ {
+ ClassWithMultipleInterfaces instance;
+ ChatMessage message;
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ instance = new ClassWithMultipleInterfaces();
+ message = new ChatMessage();
+ }
+
+ [Benchmark]
+ public int QueryDefaultInterface()
+ {
+ return instance.DefaultIntProperty;
+ }
+
+ [Benchmark]
+ public int QueryNonDefaultInterface()
+ {
+ return instance.IntProperty;
+ }
+
+ [Benchmark]
+ public bool QueryNonDefaultInterface2()
+ {
+ return instance.BoolProperty;
+ }
+
+ [Benchmark]
+ public void QueryDefaultInterfaceSetProperty()
+ {
+ instance.DefaultIntProperty = 4;
+ }
+
+ [Benchmark]
+ public void QueryNonDefaultInterfaceSetProperty()
+ {
+ instance.IntProperty = 4;
+ }
+
+ [Benchmark]
+ public bool QuerySDKDefaultInterface()
+ {
+ return message.IsForwardingDisabled;
+ }
+
+ [Benchmark]
+ public bool QuerySDKNonDefaultInterface()
+ {
+ return message.IsSeen;
+ }
+
+ // The following 2 benchmarks try to benchmark the time taken for the first call
+ // rather than the mean time over several calls. It has the overhead of the object
+ // construction, but it can be used to track regressions to performance.
+ [Benchmark]
+ public int ConstructAndQueryDefaultInterfaceFirstCall()
+ {
+ ClassWithMultipleInterfaces instance2 = new ClassWithMultipleInterfaces();
+ return instance2.DefaultIntProperty;
+ }
+
+ [Benchmark]
+ public int ConstructAndQueryNonDefaultInterfaceFirstCall()
+ {
+ ClassWithMultipleInterfaces instance2 = new ClassWithMultipleInterfaces();
+ return instance2.IntProperty;
+ }
+ }
+}

Directory.Build.props

@@ -42,6 +42,7 @@
 
  <PropertyGroup Condition="'$(MSBuildProjectExtension)' == '.csproj'">
  <TargetPlatformIdentifier>Windows</TargetPlatformIdentifier>
+ <TargetPlatformVersion>10.0.18362.0</TargetPlatformVersion>
  </PropertyGroup>
 
  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">

Projections/Benchmark/Benchmark.csproj

@@ -0,0 +1,65 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFrameworks>netstandard2.0;net5.0</TargetFrameworks>
+ <Platforms>x64;x86</Platforms>
+ <LangVersion>8</LangVersion>
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <GenerateTestProjection Condition="'$(GenerateTestProjection)$(Configuration)' == 'Release'">true</GenerateTestProjection>
+ <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+ <NoWarn>8305;0618</NoWarn>
+ </PropertyGroup>
+
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <DebugType>full</DebugType>
+ <DebugSymbols>true</DebugSymbols>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="System.Numerics.Vectors" Version="4.5.0" />
+ <ProjectReference Include="..\..\TestWinRT\BenchmarkComponent\BenchmarkComponent.vcxproj" />
+ <ProjectReference Include="..\..\WinRT.Runtime\WinRT.Runtime.csproj" />
+ <ProjectReference Include="..\..\cswinrt\cswinrt.vcxproj" />
+ <ProjectReference Include="..\Windows\Windows.csproj" />
+ </ItemGroup>
+
+ <Target Name="GenerateProjection" Condition="'$(GenerateTestProjection)' == 'true'">
+ <ItemGroup>
+ <ReferenceWinMDs Include="@(ReferencePath)" Condition="'%(ReferencePath.WinMDFile)' == 'true'" />
+ <!--Do not publish projection source winmds -->
+ <ReferenceCopyLocalPaths Remove="@(ReferenceCopyLocalPaths)" Condition="%(ReferenceCopyLocalPaths.Filename) == 'BenchmarkComponent'" />
+ </ItemGroup>
+ <PropertyGroup>
+ <CsWinRTVerbosity>high</CsWinRTVerbosity>
+ <CsWinRTResponseFile>$(GeneratedFilesDir)cswinrt_benchmark.rsp</CsWinRTResponseFile>
+ <CsWinRTCommand>$(CsWinRTExe) %40"$(CsWinRTResponseFile)"</CsWinRTCommand>
+ </PropertyGroup>
+ <PropertyGroup>
+ <CsWinRTParams>
+-verbose
+-in 10.0.18362.0
+-in @(ReferenceWinMDs->'"%(FullPath)"', ' ')
+-out "$(GeneratedFilesDir.TrimEnd('\'))"
+-exclude Windows
+-include BenchmarkComponent
+ </CsWinRTParams>
+ </PropertyGroup>
+ <MakeDir Directories="$(GeneratedFilesDir)" />
+ <WriteLinesToFile File="$(CsWinRTResponseFile)" Lines="$(CsWinRTParams)" Overwrite="true" WriteOnlyWhenDifferent="true" />
+ <Message Text="$(CsWinRTCommand)" Importance="$(CsWinRTVerbosity)" />
+ <Exec Command="$(CsWinRTCommand)" />
+ </Target>
+
+ <Target Name="IncludeProjection" DependsOnTargets="GenerateProjection" BeforeTargets="CoreCompile">
+ <ItemGroup>
+ <Compile Include="$(GeneratedFilesDir)*.cs" Exclude="@(Compile)" />
+ <!--Remove references to projection source winmds to prevent compile conflict warnings-->
+ <ReferencePathWithRefAssemblies Remove="@(ReferencePathWithRefAssemblies)" Condition="%(ReferencePathWithRefAssemblies.Filename) == 'BenchmarkComponent'" />
+ <!--Also remove ReferencePath winmds to prevent error NETSDK1130 false positive-->
+ <ReferencePath Remove="@(ReferencePath)" Condition="%(ReferencePath.Filename) == 'BenchmarkComponent'" />
+ </ItemGroup>
+ </Target>
+
+</Project>

Projections/Test/Test.csproj

@@ -3,7 +3,6 @@
  <PropertyGroup>
  <TargetFrameworks>netstandard2.0;net5.0</TargetFrameworks>
  <Platforms>x64;x86</Platforms>
- <TargetPlatformVersion>10.0.18362.0</TargetPlatformVersion>
  <LangVersion>8</LangVersion>
  </PropertyGroup>
 

Projections/WinUI/WinUI.csproj

@@ -3,7 +3,6 @@
  <PropertyGroup>
  <TargetFrameworks>netstandard2.0;net5.0</TargetFrameworks>
  <Platforms>x64;x86</Platforms>
- <TargetPlatformVersion>10.0.18362.0</TargetPlatformVersion>
  <LangVersion>8</LangVersion>
  </PropertyGroup>
 

Projections/Windows/Windows.csproj

@@ -3,7 +3,6 @@
  <PropertyGroup>
  <TargetFrameworks>netstandard2.0;net5.0</TargetFrameworks>
  <Platforms>x64;x86</Platforms>
- <TargetPlatformVersion>10.0.18362.0</TargetPlatformVersion>
  <LangVersion>8</LangVersion>
  </PropertyGroup>
 

README.md

@@ -117,12 +117,16 @@ The **/TestComponentCSharp** folder contains an implementation of a WinRT test c
 
 ## /Projections
 
-The **/Projections** folder contains several projects for generating and building projections from the Windows SDK, WinUI, and Test metadata (produced by the TestWinRT and TestComponentCSharp projects).
+The **/Projections** folder contains several projects for generating and building projections from the Windows SDK, WinUI, Benchmark (produced by the BenchmarkComponent project), and Test metadata (produced by the TestWinRT and TestComponentCSharp projects).
 
 ## /UnitTest
 
 The **/UnitTest** folder contains unit tests for validating the Windows SDK, WinUI, and Test projections generated above. All pull requests should ensure that this project executes without errors.
 
+## /Benchmarks
+
+The **/Benchmarks** folder contains benchmarks written using BenchmarkDotNet to track the performance of scenarios in the generated projection. To run the benchmarks using the CsWinRT projection, run **benchmark.cmd**. To run the same benchmarks using the built-in WinMD support in NET Core 3.1 to compare against as a baseline, run **benchmark_winmd.cmd**.
+
 ## /WinUIDesktopSample
 
 The **/WinUIDesktopSample** contains an end-to-end sample app that uses the Windows SDK and WinUI projections generated above.

benchmark.cmd

@@ -0,0 +1,2 @@
+msbuild Benchmarks\Benchmarks.csproj -t:restore -t:build /p:platform=x64 /p:configuration=release
+dotnet %~dp0Benchmarks\bin\x64\Release\netcoreapp2.0\Benchmarks.dll -filter * --runtimes netcoreapp2.0 netcoreapp3.1 netcoreapp5.0

benchmark_winmd.cmd

@@ -0,0 +1,4 @@
+msbuild Benchmarks\Benchmarks.csproj -t:restore -t:clean;rebuild /p:BenchmarkWinmdSupport=true /p:platform=x64 /p:configuration=release /p:TargetFramework=netcoreapp3.1
+%~dp0Benchmarks\bin\x64\Release\netcoreapp3.1\Benchmarks.exe -filter *
+rem Clean project to prevent mismatch scenarios with the typical benchmark.cmd scenario.
+msbuild Benchmarks\Benchmarks.csproj -t:restore -t:clean /p:BenchmarkWinmdSupport=true /p:platform=x64 /p:configuration=release /p:TargetFramework=netcoreapp3.1 >nul