Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial ScottPlotExporter with just Bar Plot and Unit Tests #2560

Merged
merged 6 commits into from
Jul 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions BenchmarkDotNet.sln
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Diagnostics.dotMemory", "src\BenchmarkDotNet.Diagnostics.dotMemory\BenchmarkDotNet.Diagnostics.dotMemory.csproj", "{2E2283A3-6DA6-4482-8518-99D6D9F689AB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting", "src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj", "{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -149,6 +153,14 @@ Global
{2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.Build.0 = Release|Any CPU
{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.Build.0 = Release|Any CPU
{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -176,6 +188,8 @@ Global
{AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9}
{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\build\common.props" />
<PropertyGroup>
<AssemblyTitle>BenchmarkDotNet plotting export support.</AssemblyTitle>
<TargetFrameworks>netstandard2.0</TargetFrameworks>
<AssemblyName>BenchmarkDotNet.Exporters.Plotting</AssemblyName>
<PackageId>BenchmarkDotNet.Exporters.Plotting</PackageId>
<RootNamespace>BenchmarkDotNet.Exporters.Plotting</RootNamespace>
<!-- needed for docfx xref resolver -->
<ProduceReferenceAssembly>True</ProduceReferenceAssembly>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\BenchmarkDotNet\BenchmarkDotNet.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="ScottPlot" Version="5.0.25" />
</ItemGroup>
</Project>
256 changes: 256 additions & 0 deletions src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Properties;
using BenchmarkDotNet.Reports;
using ScottPlot;
using ScottPlot.Plottables;

namespace BenchmarkDotNet.Exporters.Plotting
{
/// <summary>
/// Provides plot exports as .png files.
/// </summary>
public class ScottPlotExporter : IExporter
{
/// <summary>
/// Default instance of the exporter with default configuration.
/// </summary>
public static readonly IExporter Default = new ScottPlotExporter();

/// <summary>
/// Gets the name of the Exporter type.
/// </summary>
public string Name => nameof(ScottPlotExporter);

/// <summary>
/// Initializes a new instance of ScottPlotExporter.
/// </summary>
/// <param name="width">The width of all plots in pixels (optional). Defaults to 1920.</param>
/// <param name="height">The height of all plots in pixels (optional). Defaults to 1080.</param>
public ScottPlotExporter(int width = 1920, int height = 1080)
{
this.Width = width;
this.Height = height;
this.IncludeBarPlot = true;
this.RotateLabels = true;
}

/// <summary>
/// Gets or sets the width of all plots in pixels.
/// </summary>
public int Width { get; set; }

/// <summary>
/// Gets or sets the height of all plots in pixels.
/// </summary>
public int Height { get; set; }

/// <summary>
/// Gets or sets a value indicating whether labels for Plot X-axis should be rotated.
/// This allows for longer labels at the expense of chart height.
/// </summary>
public bool RotateLabels { get; set; }

/// <summary>
/// Gets or sets a value indicating whether a bar plot for time-per-op
/// measurement values should be exported.
/// </summary>
public bool IncludeBarPlot { get; set; }

/// <summary>
/// Not supported.
/// </summary>
/// <param name="summary">This parameter is not used.</param>
/// <param name="logger">This parameter is not used.</param>
/// <exception cref="NotSupportedException"></exception>
public void ExportToLog(Summary summary, ILogger logger)
{
throw new NotSupportedException();
}

/// <summary>
/// Exports plots to .png file.
/// </summary>
/// <param name="summary">The summary to be exported.</param>
/// <param name="consoleLogger">Logger to output to.</param>
/// <returns>The file paths of every plot exported.</returns>
public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
{
var title = summary.Title;
var version = BenchmarkDotNetInfo.Instance.BrandTitle;
var annotations = GetAnnotations(version);

var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements));

foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name))
{
var benchmarkName = benchmark.Key;

// Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param].
var timeStats = from report in benchmark
let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty)
from measurement in report.AllMeasurements
let measurementValue = measurement.Nanoseconds / measurement.Operations
group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g
select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList()));

if (this.IncludeBarPlot)
{
// <BenchmarkName>-barplot.png
yield return CreateBarPlot(
$"{title} - {benchmarkName}",
Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"),
$"Time ({timeUnit})",
"Target",
timeStats,
annotations);
}

/* TODO: Rest of the RPlotExporter plots.
<BenchmarkName>-boxplot.png
<BenchmarkName>-<MethodName>-density.png
<BenchmarkName>-<MethodName>-facetTimeline.png
<BenchmarkName>-<MethodName>-facetTimelineSmooth.png
<BenchmarkName>-<MethodName>-<JobName>-timelineSmooth.png
<BenchmarkName>-<MethodName>-<JobName>-timelineSmooth.png*/
}
}

/// <summary>
/// Calculate Standard Deviation.
/// </summary>
/// <param name="values">Values to calculate from.</param>
/// <returns>Standard deviation of values.</returns>
private static double StandardError(IReadOnlyList<double> values)
{
double average = values.Average();
double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum();
double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count);
return standardDeviation / Math.Sqrt(values.Count);
}

/// <summary>
/// Gets the lowest appropriate time scale across all measurements.
/// </summary>
/// <param name="values">All measurements</param>
/// <returns>A unit and scaling factor to convert from nanoseconds.</returns>
private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable<Measurement> values)
{
var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min();
if (minValue > 1000000000d)
{
return ("sec", 1000000000d);
}

if (minValue > 1000000d)
{
return ("ms", 1000000d);
}

if (minValue > 1000d)
{
return ("us", 1000d);
}

return ("ns", 1d);
}

private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList<Annotation> annotations)
{
Plot plt = new Plot();
plt.Title(title, 28);
plt.YLabel(yLabel);
plt.XLabel(xLabel);

var palette = new ScottPlot.Palettes.Category10();

var legendPalette = data.Select(d => d.JobId)
.Distinct()
.Select((jobId, index) => (jobId, index))
.ToDictionary(t => t.jobId, t => palette.GetColor(t.index));

plt.Legend.IsVisible = true;
plt.Legend.Location = Alignment.UpperRight;
var legend = data.Select(d => d.JobId)
.Distinct()
.Select((label, index) => new LegendItem()
{
Label = label,
FillColor = legendPalette[label]
})
.ToList();

plt.Legend.ManualItems.AddRange(legend);

var jobCount = plt.Legend.ManualItems.Count;
var ticks = data
.Select((d, index) => new Tick(index, d.Target))
.ToArray();
plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
plt.Axes.Bottom.MajorTickStyle.Length = 0;

if (this.RotateLabels)
{
plt.Axes.Bottom.TickLabelStyle.Rotation = 45;
plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft;

// determine the width of the largest tick label
float largestLabelWidth = 0;
foreach (Tick tick in ticks)
{
PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label);
largestLabelWidth = Math.Max(largestLabelWidth, size.Width);
}

// ensure axis panels do not get smaller than the largest label
plt.Axes.Bottom.MinimumSize = largestLabelWidth;
plt.Axes.Right.MinimumSize = largestLabelWidth;
}

var bars = data
.Select((d, index) => new Bar()
{
Position = ticks[index].Position,
Value = d.Mean,
Error = d.StdError,
FillColor = legendPalette[d.JobId]
});
plt.Add.Bars(bars);

// Tell the plot to autoscale with no padding beneath the bars
plt.Axes.Margins(bottom: 0, right: .2);

plt.PlottableList.AddRange(annotations);

plt.SavePng(fileName, this.Width, this.Height);
return Path.GetFullPath(fileName);
}

/// <summary>
/// Provides a list of annotations to put over the data area.
/// </summary>
/// <param name="version">The version to be displayed.</param>
/// <returns>A list of annotations for every plot.</returns>
private IReadOnlyList<Annotation> GetAnnotations(string version)
{
var versionAnnotation = new Annotation()
{
Label =
{
Text = version,
FontSize = 14,
ForeColor = new Color(0, 0, 0, 100)
},
OffsetY = 10,
OffsetX = 20,
Alignment = Alignment.LowerRight
};


return new[] { versionAnnotation };
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\build\common.props" />
<PropertyGroup>
<TargetFrameworks>net8.0;net462</TargetFrameworks>
timcassell marked this conversation as resolved.
Show resolved Hide resolved
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
<GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
<GenerateDocumentationFile>false</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
<PackageReference Include="Microsoft.NETCore.Platforms" Version="6.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj" />
<ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
<ProjectReference Include="..\BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="xunit" Version="2.6.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Loading
Loading