Reorganized benchmarks

EFCore.sln

@@ -131,7 +131,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.SqlServer.Benchmarks
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Sqlite.Benchmarks", "benchmark\EFCore.Sqlite.Benchmarks\EFCore.Sqlite.Benchmarks.csproj", "{5A44F67E-517F-458D-B9C7-12A4DBBAD64A}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Benchmarks.SqlServer.EF6", "benchmark\EFCore.SqlServer.EF6.Benchmarks\EFCore.Benchmarks.SqlServer.EF6.csproj", "{678AB38D-B27C-4690-A3F9-2D2488391658}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EF6.SqlServer.Benchmarks", "benchmark\EF6.SqlServer.Benchmarks\EF6.SqlServer.Benchmarks.csproj", "{678AB38D-B27C-4690-A3F9-2D2488391658}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EFCore.Benchmarks", "benchmark\EFCore.Benchmarks\EFCore.Benchmarks.csproj", "{2642F4F0-69BE-4C43-94B7-B298FEC87D89}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EF.Benchmarks.Shared", "benchmark\EF.Benchmarks.Shared\EF.Benchmarks.Shared.csproj", "{BFC26566-4C6D-4904-A559-8FFE09369901}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -355,6 +359,14 @@ Global
 		{678AB38D-B27C-4690-A3F9-2D2488391658}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{678AB38D-B27C-4690-A3F9-2D2488391658}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{678AB38D-B27C-4690-A3F9-2D2488391658}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2642F4F0-69BE-4C43-94B7-B298FEC87D89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2642F4F0-69BE-4C43-94B7-B298FEC87D89}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2642F4F0-69BE-4C43-94B7-B298FEC87D89}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2642F4F0-69BE-4C43-94B7-B298FEC87D89}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BFC26566-4C6D-4904-A559-8FFE09369901}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BFC26566-4C6D-4904-A559-8FFE09369901}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BFC26566-4C6D-4904-A559-8FFE09369901}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BFC26566-4C6D-4904-A559-8FFE09369901}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -414,6 +426,8 @@ Global
 		{24D7B6D5-A22A-4734-BB55-F464C112F891} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
 		{5A44F67E-517F-458D-B9C7-12A4DBBAD64A} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
 		{678AB38D-B27C-4690-A3F9-2D2488391658} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
+		{2642F4F0-69BE-4C43-94B7-B298FEC87D89} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
+		{BFC26566-4C6D-4904-A559-8FFE09369901} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {285A5EB4-BCF4-40EB-B9E1-DF6DBCB5E705}

benchmark/EF.Benchmarks.Shared/EF.Benchmarks.Shared.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net461;netstandard2.0</TargetFrameworks>
+    <RootNamespace>Microsoft.EntityFrameworkCore.Benchmarks</RootNamespace>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="$(Configuration.StartsWith('Release'))">
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+
+</Project>

benchmark/EF.Benchmarks.Shared/EFCoreBenchmarkRunner.cs

@@ -0,0 +1,54 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Exporters.Csv;
+using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+
+namespace Microsoft.EntityFrameworkCore.Benchmarks
+{
+    public static class EFCoreBenchmarkRunner
+    {
+        public static void Run(string[] args, Assembly assembly, IConfig config = null)
+        {
+            if (config == null)
+            {
+                config = DefaultConfig.Instance;
+            }
+
+            config = config.With(DefaultConfig.Instance.GetDiagnosers().Concat(new[] { MemoryDiagnoser.Default }).ToArray());
+
+            var index = Array.FindIndex(args, s => s == "--perflab");
+            if (index >= 0)
+            {
+                var argList = args.ToList();
+                argList.RemoveAt(index);
+                args = argList.ToArray();
+
+                config = config
+                    .With(StatisticColumn.OperationsPerSecond, new ParamsSummaryColumn())
+                    .With(
+                        MarkdownExporter.GitHub, new CsvExporter(
+                        CsvSeparator.Comma,
+                        new SummaryStyle
+                        {
+                            PrintUnitsInHeader = true,
+                            PrintUnitsInContent = false,
+                            TimeUnit = TimeUnit.Microsecond,
+                            SizeUnit = SizeUnit.KB
+                        }));
+            }
+
+            BenchmarkSwitcher.FromAssembly(assembly).Run(args, config);
+        }
+    }
+}

benchmark/EFCore.SqlServer.EF6.Benchmarks/EFCore.Benchmarks.SqlServer.EF6.csproj → benchmark/EF6.SqlServer.Benchmarks/EF6.SqlServer.Benchmarks.csproj

@@ -18,7 +18,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Compile Include="..\Shared\**\*.cs" Link="%(RecursiveDir)%(FileName)%(Extension)" />
+    <ProjectReference Include="..\EF.Benchmarks.Shared\EF.Benchmarks.Shared.csproj" />
   </ItemGroup>
 
 </Project>

benchmark/EFCore.SqlServer.EF6.Benchmarks/Models/AdventureWorks/AdventureWorksFixture.cs → benchmark/EF6.SqlServer.Benchmarks/Models/AdventureWorks/AdventureWorksFixture.cs

@@ -1,11 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using Microsoft.EntityFrameworkCore.Benchmarks;
+
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.AdventureWorks
 {
     public static class AdventureWorksFixture
     {
-        public static string ConnectionString { get; } = $"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database=AdventureWorks2014;";
+        public static readonly string ConnectionString = SqlServerBenchmarkEnvironment.CreateConnectionString("AdventureWorks2014");
 
         // This method is called from timed code, be careful when changing it
         public static AdventureWorksContext CreateContext()

benchmark/EFCore.SqlServer.EF6.Benchmarks/Models/Orders/OrdersFixture.cs → benchmark/EF6.SqlServer.Benchmarks/Models/Orders/OrdersFixture.cs

@@ -19,7 +19,7 @@ public class OrdersFixture : OrdersFixtureSeedBase
         public OrdersFixture(string databaseName, int productCount, int customerCount,
             int ordersPerCustomer, int linesPerOrder, Action<DbContext> seedAction = null)
         {
-            _connectionString = $"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database={databaseName};";
+            _connectionString = SqlServerBenchmarkEnvironment.CreateConnectionString(databaseName);
             _productCount = productCount;
             _customerCount = customerCount;
             _ordersPerCustomer = ordersPerCustomer;

benchmark/EFCore.SqlServer.EF6.Benchmarks/Extensions.cs → benchmark/EF6.SqlServer.Benchmarks/Support/Extensions.cs

@@ -10,6 +10,7 @@
 
 // ReSharper disable PossibleNullReferenceException
 
+// ReSharper disable once CheckNamespace
 namespace Microsoft.EntityFrameworkCore.Benchmarks
 {
     public static class Extensions

benchmark/EF6.SqlServer.Benchmarks/Support/Program.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Exporters.Csv;
+using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+
+// ReSharper disable once CheckNamespace
+namespace Microsoft.EntityFrameworkCore.Benchmarks
+{
+    public class Program
+    {
+        private static void Main(string[] args) => EFCoreBenchmarkRunner.Run(args, typeof(Program).Assembly);
+    }
+}

benchmark/EF6.SqlServer.Benchmarks/Support/SqlServerBenchmarkEnvironment.cs

@@ -0,0 +1,34 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.IO;
+using System.Data.SqlClient;
+using Microsoft.Extensions.Configuration;
+
+// ReSharper disable once CheckNamespace
+namespace Microsoft.EntityFrameworkCore.Benchmarks
+{
+    public static class SqlServerBenchmarkEnvironment
+    {
+        public static IConfiguration Config { get; }
+
+        static SqlServerBenchmarkEnvironment()
+        {
+            var configBuilder = new ConfigurationBuilder()
+                .SetBasePath(Directory.GetCurrentDirectory())
+                .AddJsonFile("config.json", optional: true)
+                .AddEnvironmentVariables();
+
+            Config = configBuilder.Build()
+                .GetSection("Test:SqlServer");
+        }
+
+        private const string DefaultConnectionString
+            = "Data Source=(localdb)\\MSSQLLocalDB;Database=master;Integrated Security=True;Connect Timeout=30;ConnectRetryCount=0";
+
+        public static string DefaultConnection => Config["DefaultConnection"] ?? DefaultConnectionString;
+
+        public static string CreateConnectionString(string name, string fileName = null, bool? multipleActiveResultSets = null)
+            => new SqlConnectionStringBuilder(DefaultConnection) { InitialCatalog = name }.ToString();
+    }
+}

benchmark/EF6.SqlServer.Benchmarks/config.json

@@ -0,0 +1,5 @@
+{
+  "benchmarks": {
+    "benchmarkDatabase": "Server=(localdb)\\mssqllocaldb;Trusted_Connection=True;MultipleActiveResultSets=true;"
+  }
+}

benchmark/EFCore.Benchmarks/EFCore.Benchmarks.csproj

@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net461;netcoreapp2.0;netcoreapp2.1;netcoreapp2.2;netcoreapp3.0</TargetFrameworks>
+    <TargetFrameworks Condition=" '$(Configuration)' == 'Release' Or '$(Configuration)' == 'Debug' ">netcoreapp3.0</TargetFrameworks>
+    <RootNamespace>Microsoft.EntityFrameworkCore.Benchmarks</RootNamespace>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="$(Configuration.StartsWith('Release'))">
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\EF.Benchmarks.Shared\EF.Benchmarks.Shared.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release' Or '$(Configuration)' == 'Debug' ">
+    <ProjectReference Include="..\..\src\EFCore.Relational\EFCore.Relational.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release22' Or '$(Configuration)' == 'Debug22' ">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="2.2.4" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release21' Or '$(Configuration)' == 'Debug21' ">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="2.1.11" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release20' Or '$(Configuration)' == 'Debug20' ">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="2.0.3" />
+  </ItemGroup>
+
+  <PropertyGroup Condition=" '$(Configuration)' == 'Release22' Or '$(Configuration)' == 'Release21' Or '$(Configuration)' == 'Release20' Or '$(Configuration)' == 'Debug22' Or '$(Configuration)' == 'Debug21' Or '$(Configuration)' == 'Debug20'">
+    <DefineConstants>$(DefineConstants);OLD_FROM_SQL</DefineConstants>
+  </PropertyGroup>
+
+</Project>

benchmark/EFCore.SqlServer.Benchmarks/ChangeTracker/DbSetOperationSqlServerTests.cs

@@ -13,15 +13,15 @@ public class AddDataVariations : AddDataVariationsBase
         {
             public override OrdersFixtureBase CreateFixture()
             {
-                return new OrdersFixture("Perf_ChangeTracker_DbSetOperation");
+                return new OrdersSqlServerFixture("Perf_ChangeTracker_DbSetOperation");
             }
         }
 
         public class ExistingDataVariations : ExistingDataVariationsBase
         {
             public override OrdersFixtureBase CreateFixture()
             {
-                return new OrdersFixture("Perf_ChangeTracker_DbSetOperation");
+                return new OrdersSqlServerFixture("Perf_ChangeTracker_DbSetOperation");
             }
         }
     }

benchmark/EFCore.SqlServer.Benchmarks/ChangeTracker/FixupSqlServerTests.cs

@@ -13,15 +13,15 @@ public class ChildVariations : ChildVariationsBase
         {
             public override OrdersFixtureBase CreateFixture()
             {
-                return new OrdersFixture("Perf_ChangeTracker_Fixup");
+                return new OrdersSqlServerFixture("Perf_ChangeTracker_Fixup");
             }
         }
 
         public class ParentVariations : ParentVariationsBase
         {
             public override OrdersFixtureBase CreateFixture()
             {
-                return new OrdersFixture("Perf_ChangeTracker_Fixup");
+                return new OrdersSqlServerFixture("Perf_ChangeTracker_Fixup");
             }
         }
     }

benchmark/EFCore.SqlServer.Benchmarks/EFCore.SqlServer.Benchmarks.csproj

@@ -15,22 +15,18 @@
     <ProjectReference Include="..\..\src\EFCore.SqlServer\EFCore.SqlServer.csproj" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(Configuration)' == 'Release22' ">
+  <ItemGroup Condition=" '$(Configuration)' == 'Release22' Or '$(Configuration)' == 'Debug22' ">
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="2.2.4" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(Configuration)' == 'Release21' ">
+  <ItemGroup Condition=" '$(Configuration)' == 'Release21' Or '$(Configuration)' == 'Debug21' ">
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="2.1.11" />
   </ItemGroup>
 
-  <ItemGroup Condition=" '$(Configuration)' == 'Release20' ">
+  <ItemGroup Condition=" '$(Configuration)' == 'Release20' Or '$(Configuration)' == 'Debug20' ">
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="2.0.3" />
   </ItemGroup>
 
-  <PropertyGroup Condition=" '$(Configuration)' == 'Release22' Or '$(Configuration)' == 'Release21' Or '$(Configuration)' == 'Release20' ">
-    <DefineConstants>$(DefineConstants);OLD_FROM_SQL</DefineConstants>
-  </PropertyGroup>
-
   <ItemGroup>
     <None Update="config.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -39,8 +35,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Compile Include="..\Shared\**\*.cs" Link="%(RecursiveDir)%(FileName)%(Extension)" />
-    <Compile Include="..\Shared.EFCore\**\*.cs" Link="%(RecursiveDir)%(FileName)%(Extension)" />
+    <ProjectReference Include="..\EFCore.Benchmarks\EFCore.Benchmarks.csproj" />
   </ItemGroup>
 
 </Project>

benchmark/EFCore.SqlServer.Benchmarks/Initialization/ColdStartEnabledSqlServerTest.cs

@@ -9,7 +9,7 @@ public class ColdStartEnabledSqlServerTest : ColdStartEnabledTests
     {
         protected override AdventureWorksContextBase CreateContext()
         {
-            return AdventureWorksFixture.CreateContext();
+            return AdventureWorksSqlServerFixture.CreateContext();
         }
     }
 }

benchmark/EFCore.SqlServer.Benchmarks/Models/AdventureWorks/AdventureWorksContext.cs → benchmark/EFCore.SqlServer.Benchmarks/Models/AdventureWorks/AdventureWorksSqlServerContext.cs

@@ -4,11 +4,11 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.AdventureWorks
 {
-    public class AdventureWorksContext : AdventureWorksContextBase
+    public class AdventureWorksSqlServerContext : AdventureWorksContextBase
     {
         private readonly string _connectionString;
 
-        public AdventureWorksContext(string connectionString)
+        public AdventureWorksSqlServerContext(string connectionString)
         {
             _connectionString = connectionString;
         }

benchmark/EFCore.SqlServer.Benchmarks/Models/AdventureWorks/AdventureWorksFixture.cs → benchmark/EFCore.SqlServer.Benchmarks/Models/AdventureWorks/AdventureWorksSqlServerFixture.cs

@@ -3,14 +3,14 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.AdventureWorks
 {
-    public static class AdventureWorksFixture
+    public static class AdventureWorksSqlServerFixture
     {
-        private static readonly string _connectionString = $"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database=AdventureWorks2014;";
+        private static readonly string _connectionString = SqlServerBenchmarkEnvironment.CreateConnectionString("AdventureWorks2014");
 
         // This method is called from timed code, be careful when changing it
         public static AdventureWorksContextBase CreateContext()
         {
-            return new AdventureWorksContext(_connectionString);
+            return new AdventureWorksSqlServerContext(_connectionString);
         }
     }
 }

benchmark/EFCore.SqlServer.Benchmarks/Models/Orders/OrdersContext.cs → benchmark/EFCore.SqlServer.Benchmarks/Models/Orders/OrdersSqlServerContext.cs

@@ -6,12 +6,12 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.Orders
 {
-    public class OrdersContext : OrdersContextBase
+    public class OrdersSqlServerContext : OrdersContextBase
     {
         private readonly string _connectionString;
         private readonly bool _disableBatching;
 
-        public OrdersContext(string connectionString, IServiceProvider serviceProvider = null, bool disableBatching = false)
+        public OrdersSqlServerContext(string connectionString, IServiceProvider serviceProvider = null, bool disableBatching = false)
             : base(serviceProvider)
         {
             _connectionString = connectionString;

benchmark/EFCore.SqlServer.Benchmarks/Models/Orders/OrdersFixture.cs → benchmark/EFCore.SqlServer.Benchmarks/Models/Orders/OrdersSqlServerFixture.cs

@@ -5,18 +5,18 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.Orders
 {
-    public class OrdersFixture : OrdersFixtureBase
+    public class OrdersSqlServerFixture : OrdersFixtureBase
     {
         private readonly string _connectionString;
 
-        public OrdersFixture(string databaseName)
+        public OrdersSqlServerFixture(string databaseName)
         {
-            _connectionString = $@"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database={databaseName};";
+            _connectionString = SqlServerBenchmarkEnvironment.CreateConnectionString(databaseName);
         }
 
         public override OrdersContextBase CreateContext(IServiceProvider serviceProvider = null, bool disableBatching = false)
         {
-            return new OrdersContext(_connectionString, serviceProvider, disableBatching);
+            return new OrdersSqlServerContext(_connectionString, serviceProvider, disableBatching);
         }
     }
 }