-
Notifications
You must be signed in to change notification settings - Fork 570
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[C#] Add initial Benchmarkdotnet tests (#751)
* Add a couple BenchmarkDotNet tests * add BenchmarkDotNet README Co-authored-by: Badrish Chandramouli <badrishc@microsoft.com>
- Loading branch information
Showing
8 changed files
with
241 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -199,3 +199,5 @@ nativebin/ | |
| cs/.idea/ | ||
| cs/remote/.idea | ||
| cs/libdpr/.idea | ||
|
|
||
| cs/**/BenchmarkDotNet.Artifacts/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
cs/performance/BenchmarkDotNet/BenchmarkDotNetTests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| </PropertyGroup> | ||
| <PropertyGroup> | ||
| <PlatformTarget>AnyCPU</PlatformTarget> | ||
| <DebugType>pdbonly</DebugType> | ||
| <DebugSymbols>true</DebugSymbols> | ||
| <AllowUnsafeBlocks>true</AllowUnsafeBlocks> | ||
| <Optimize>true</Optimize> | ||
| <Configuration>Release</Configuration> | ||
| <IsPackable>false</IsPackable> | ||
| <ApplicationIcon /> | ||
| <OutputType>Exe</OutputType> | ||
| <StartupObject /> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <PackageReference Include="BenchmarkDotNet" Version="0.13.0" /> | ||
| <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.0" /> | ||
| </ItemGroup> | ||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\src\core\FASTER.core.csproj" /> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. | ||
|
|
||
| using BenchmarkDotNet.Running; | ||
| using System.IO; | ||
|
|
||
| namespace BenchmarkDotNetTests | ||
| { | ||
| public class BenchmarkDotNetTestsApp | ||
| { | ||
| public static string TestDirectory => Path.Combine(Path.GetDirectoryName(typeof(BenchmarkDotNetTestsApp).Assembly.Location), "Tests"); | ||
|
|
||
| public static void Main(string[] args) | ||
| { | ||
| BenchmarkSwitcher.FromAssembly(typeof(BenchmarkDotNetTestsApp).Assembly).Run(args); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. | ||
|
|
||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Configs; | ||
| using FASTER.core; | ||
|
|
||
| #pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value'; happens due to [Params(..)] | ||
|
|
||
| namespace BenchmarkDotNetTests | ||
| { | ||
| [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)] | ||
| public class LightEpochTests | ||
| { | ||
| [Params(10_000_000, 100_000_000, 1_000_000_000)] | ||
| public int NumIterations; | ||
|
|
||
| [BenchmarkCategory("LightEpoch"), Benchmark] | ||
| public void AcquireAndRelease() | ||
| { | ||
| var epoch = new LightEpoch(); | ||
| for (int i = 0; i < NumIterations; i++) | ||
| { | ||
| epoch.Resume(); | ||
| epoch.Suspend(); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| ## BenchmarkDotNet tests | ||
|
|
||
| [BenchmarkDotNet](https://benchmarkdotnet.org) is a micro-benchmarking suite that supports annotations on functions, somewhat similar to unit tests. | ||
|
|
||
| To run: | ||
| - Build Release | ||
| - Execute `bin/Release/[platform]/BenchmarkDotNetTests.exe -f *LightEpoch*` to run the LightEpoch tests. See [BenchmarkDotNet](https://benchmarkdotnet.org) and related documentation for more details. | ||
|
|
||
| Currently we have two very simple benchmarks; more will be added later. | ||
| - LightEpochTests: Currently runs a microbenchmark of epoch Acquire/Release. | ||
| - SyncVsAsyncTests: Compares sync vs. async API performance. | ||
|
|
||
| BenchmarkDotNet does not support cross-run comparisons; currently, just save the output to a file and diff. A script will be written in the future to automate these comparisons. | ||
|
|
||
| ### Adding a Test | ||
| Use one of the existing tests as an example, and add an entry in this doc describing it briefly. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. | ||
|
|
||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Configs; | ||
| using FASTER.core; | ||
| using System; | ||
| using System.IO; | ||
| using System.Threading.Tasks; | ||
|
|
||
| #pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value'; happens due to [Params(..)] | ||
|
|
||
| namespace BenchmarkDotNetTests | ||
| { | ||
| [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)] | ||
| public class SyncVsAsync | ||
| { | ||
| [Params(100, 1_000_000)] | ||
| public int NumRecords; | ||
|
|
||
| //[ParamsAllValues] | ||
| //bool useAsync; | ||
|
|
||
| FasterKV<long, long> store; | ||
| IDevice logDevice; | ||
| string logDirectory; | ||
|
|
||
| void SetupStore() | ||
| { | ||
| logDirectory = BenchmarkDotNetTestsApp.TestDirectory; | ||
| var logFilename = Path.Combine(logDirectory, $"{nameof(SyncVsAsync)}_{Guid.NewGuid()}.log"); | ||
| logDevice = Devices.CreateLogDevice(logFilename, preallocateFile: true, deleteOnClose: true, useIoCompletionPort: true); | ||
| var logSettings = new LogSettings | ||
| { | ||
| LogDevice = logDevice | ||
| }; | ||
| store = new FasterKV<long, long>(1L << 20, logSettings); | ||
| } | ||
|
|
||
| void PopulateStoreSync() | ||
| { | ||
| using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>(); | ||
| for (long ii = 0; ii < NumRecords; ++ii) | ||
| session.Upsert(ii, ii); | ||
| } | ||
|
|
||
| async ValueTask PopulateStoreAsync() | ||
| { | ||
| using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>(); | ||
| for (long ii = 0; ii < NumRecords; ++ii) | ||
| { | ||
| var result = await session.UpsertAsync(ii, ii); | ||
| while (result.Status.IsPending) | ||
| result = await result.CompleteAsync().ConfigureAwait(false); | ||
| } | ||
| } | ||
|
|
||
| [GlobalSetup(Targets = new[] { nameof(InsertSync), nameof(InsertAsync) })] | ||
| public void SetupEmptyStore() => SetupStore(); | ||
|
|
||
| [GlobalSetup(Targets = new[] { nameof(RMWSync), nameof(RMWAsync), nameof(ReadSync), nameof(ReadAsync) })] | ||
| public void SetupPopulatedStore() | ||
| { | ||
| SetupStore(); | ||
| PopulateStoreSync(); | ||
| } | ||
|
|
||
| [GlobalCleanup] | ||
| public void TearDown() | ||
| { | ||
| store?.Dispose(); | ||
| store = null; | ||
| logDevice?.Dispose(); | ||
| logDevice = null; | ||
| try | ||
| { | ||
| Directory.Delete(logDirectory); | ||
| } | ||
| catch { } | ||
| } | ||
|
|
||
| [BenchmarkCategory("Insert"), Benchmark(Baseline = true)] | ||
| public void InsertSync() => PopulateStoreSync(); | ||
|
|
||
| [BenchmarkCategory("Insert"), Benchmark] | ||
| public ValueTask InsertAsync() => PopulateStoreAsync(); | ||
|
|
||
| [BenchmarkCategory("RMW"), Benchmark(Baseline = true)] | ||
| public void RMWSync() | ||
| { | ||
| using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>(); | ||
| for (long ii = 0; ii < NumRecords; ++ii) | ||
| session.RMW(ii, ii * 2); | ||
| session.CompletePending(); | ||
| } | ||
|
|
||
| [BenchmarkCategory("RMW"), Benchmark] | ||
| public async ValueTask RMWAsync() | ||
| { | ||
| using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>(); | ||
| for (long ii = 0; ii < NumRecords; ++ii) | ||
| { | ||
| var result = await session.RMWAsync(ii, ii * 2); | ||
| while (result.Status.IsPending) | ||
| result = await result.CompleteAsync().ConfigureAwait(false); | ||
| } | ||
| } | ||
|
|
||
| [BenchmarkCategory("Read"), Benchmark(Baseline = true)] | ||
| public void ReadSync() | ||
| { | ||
| using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>(); | ||
| for (long ii = 0; ii < NumRecords; ++ii) | ||
| { | ||
| var (status, output) = session.Read(ii); | ||
| if (status.IsPending) | ||
| { | ||
| session.CompletePendingWithOutputs(out var completedOutputs, wait: true); | ||
| completedOutputs.Dispose(); | ||
| } | ||
| } | ||
| session.CompletePending(); | ||
| } | ||
|
|
||
| [BenchmarkCategory("Read"), Benchmark] | ||
| public async ValueTask ReadAsync() | ||
| { | ||
| using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>(); | ||
| for (long ii = 0; ii < NumRecords; ++ii) | ||
| { | ||
| var (status, output) = (await session.ReadAsync(ii).ConfigureAwait(false)).Complete(); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters