Initial ScottPlotExporter with just Bar Plot and Unit Tests (#2560)

* Initial ScottPlotExporter with just Bar Plot and Unit Tests

* Simplifying project settings, added missing common.props, adde some documentation for config settings.

* Removed redundant warning suppressions

* Fix missing public documentation

* Removed redundant condition

* Update tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj

---------

Co-authored-by: Tim Cassell <35501420+timcassell@users.noreply.github.com>

BenchmarkDotNet.sln

@@ -55,6 +55,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Diagnostics.dotMemory", "src\BenchmarkDotNet.Diagnostics.dotMemory\BenchmarkDotNet.Diagnostics.dotMemory.csproj", "{2E2283A3-6DA6-4482-8518-99D6D9F689AB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting", "src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj", "{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
+EndProject
 Global
  GlobalSection(SolutionConfigurationPlatforms) = preSolution
  Debug|Any CPU = Debug|Any CPU
@@ -149,6 +153,14 @@ Global
  {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
  {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
  {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.Build.0 = Release|Any CPU
+ {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.Build.0 = Release|Any CPU
+ {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
  EndGlobalSection
  GlobalSection(SolutionProperties) = preSolution
  HideSolutionNode = FALSE
@@ -176,6 +188,8 @@ Global
  {AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9}
  {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
  {2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
+ {B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
+ {199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
  EndGlobalSection
  GlobalSection(ExtensibilityGlobals) = postSolution
  SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <Import Project="..\..\build\common.props" />
+ <PropertyGroup>
+ <AssemblyTitle>BenchmarkDotNet plotting export support.</AssemblyTitle>
+ <TargetFrameworks>netstandard2.0</TargetFrameworks>
+ <AssemblyName>BenchmarkDotNet.Exporters.Plotting</AssemblyName>
+ <PackageId>BenchmarkDotNet.Exporters.Plotting</PackageId>
+ <RootNamespace>BenchmarkDotNet.Exporters.Plotting</RootNamespace>
+ <!-- needed for docfx xref resolver -->
+ <ProduceReferenceAssembly>True</ProduceReferenceAssembly>
+ <Nullable>enable</Nullable>
+ </PropertyGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+ </ItemGroup>
+ <ItemGroup>
+ <PackageReference Include="ScottPlot" Version="5.0.25" />
+ </ItemGroup>
+</Project>

src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs

@@ -0,0 +1,256 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Properties;
+using BenchmarkDotNet.Reports;
+using ScottPlot;
+using ScottPlot.Plottables;
+
+namespace BenchmarkDotNet.Exporters.Plotting
+{
+ /// <summary>
+ /// Provides plot exports as .png files.
+ /// </summary>
+ public class ScottPlotExporter : IExporter
+ {
+ /// <summary>
+ /// Default instance of the exporter with default configuration.
+ /// </summary>
+ public static readonly IExporter Default = new ScottPlotExporter();
+
+ /// <summary>
+ /// Gets the name of the Exporter type.
+ /// </summary>
+ public string Name => nameof(ScottPlotExporter);
+
+ /// <summary>
+ /// Initializes a new instance of ScottPlotExporter.
+ /// </summary>
+ /// <param name="width">The width of all plots in pixels (optional). Defaults to 1920.</param>
+ /// <param name="height">The height of all plots in pixels (optional). Defaults to 1080.</param>
+ public ScottPlotExporter(int width = 1920, int height = 1080)
+ {
+ this.Width = width;
+ this.Height = height;
+ this.IncludeBarPlot = true;
+ this.RotateLabels = true;
+ }
+
+ /// <summary>
+ /// Gets or sets the width of all plots in pixels.
+ /// </summary>
+ public int Width { get; set; }
+
+ /// <summary>
+ /// Gets or sets the height of all plots in pixels.
+ /// </summary>
+ public int Height { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated.
+ /// This allows for longer labels at the expense of chart height.
+ /// </summary>
+ public bool RotateLabels { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value indicating whether a bar plot for time-per-op
+ /// measurement values should be exported.
+ /// </summary>
+ public bool IncludeBarPlot { get; set; }
+
+ /// <summary>
+ /// Not supported.
+ /// </summary>
+ /// <param name="summary">This parameter is not used.</param>
+ /// <param name="logger">This parameter is not used.</param>
+ /// <exception cref="NotSupportedException"></exception>
+ public void ExportToLog(Summary summary, ILogger logger)
+ {
+ throw new NotSupportedException();
+ }
+
+ /// <summary>
+ /// Exports plots to .png file.
+ /// </summary>
+ /// <param name="summary">The summary to be exported.</param>
+ /// <param name="consoleLogger">Logger to output to.</param>
+ /// <returns>The file paths of every plot exported.</returns>
+ public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
+ {
+ var title = summary.Title;
+ var version = BenchmarkDotNetInfo.Instance.BrandTitle;
+ var annotations = GetAnnotations(version);
+
+ var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements));
+
+ foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name))
+ {
+ var benchmarkName = benchmark.Key;
+
+ // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param].
+ var timeStats = from report in benchmark
+ let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty)
+ from measurement in report.AllMeasurements
+ let measurementValue = measurement.Nanoseconds / measurement.Operations
+ group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g
+ select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList()));
+
+ if (this.IncludeBarPlot)
+ {
+ // <BenchmarkName>-barplot.png
+ yield return CreateBarPlot(
+ $"{title} - {benchmarkName}",
+ Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"),
+ $"Time ({timeUnit})",
+ "Target",
+ timeStats,
+ annotations);
+ }
+
+ /* TODO: Rest of the RPlotExporter plots.
+ <BenchmarkName>-boxplot.png
+ <BenchmarkName>-<MethodName>-density.png
+ <BenchmarkName>-<MethodName>-facetTimeline.png
+ <BenchmarkName>-<MethodName>-facetTimelineSmooth.png
+ <BenchmarkName>-<MethodName>-<JobName>-timelineSmooth.png
+ <BenchmarkName>-<MethodName>-<JobName>-timelineSmooth.png*/
+ }
+ }
+
+ /// <summary>
+ /// Calculate Standard Deviation.
+ /// </summary>
+ /// <param name="values">Values to calculate from.</param>
+ /// <returns>Standard deviation of values.</returns>
+ private static double StandardError(IReadOnlyList<double> values)
+ {
+ double average = values.Average();
+ double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum();
+ double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count);
+ return standardDeviation / Math.Sqrt(values.Count);
+ }
+
+ /// <summary>
+ /// Gets the lowest appropriate time scale across all measurements.
+ /// </summary>
+ /// <param name="values">All measurements</param>
+ /// <returns>A unit and scaling factor to convert from nanoseconds.</returns>
+ private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable<Measurement> values)
+ {
+ var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min();
+ if (minValue > 1000000000d)
+ {
+ return ("sec", 1000000000d);
+ }
+
+ if (minValue > 1000000d)
+ {
+ return ("ms", 1000000d);
+ }
+
+ if (minValue > 1000d)
+ {
+ return ("us", 1000d);
+ }
+
+ return ("ns", 1d);
+ }
+
+ private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList<Annotation> annotations)
+ {
+ Plot plt = new Plot();
+ plt.Title(title, 28);
+ plt.YLabel(yLabel);
+ plt.XLabel(xLabel);
+
+ var palette = new ScottPlot.Palettes.Category10();
+
+ var legendPalette = data.Select(d => d.JobId)
+ .Distinct()
+ .Select((jobId, index) => (jobId, index))
+ .ToDictionary(t => t.jobId, t => palette.GetColor(t.index));
+
+ plt.Legend.IsVisible = true;
+ plt.Legend.Location = Alignment.UpperRight;
+ var legend = data.Select(d => d.JobId)
+ .Distinct()
+ .Select((label, index) => new LegendItem()
+ {
+ Label = label,
+ FillColor = legendPalette[label]
+ })
+ .ToList();
+
+ plt.Legend.ManualItems.AddRange(legend);
+
+ var jobCount = plt.Legend.ManualItems.Count;
+ var ticks = data
+ .Select((d, index) => new Tick(index, d.Target))
+ .ToArray();
+ plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
+ plt.Axes.Bottom.MajorTickStyle.Length = 0;
+
+ if (this.RotateLabels)
+ {
+ plt.Axes.Bottom.TickLabelStyle.Rotation = 45;
+ plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft;
+
+ // determine the width of the largest tick label
+ float largestLabelWidth = 0;
+ foreach (Tick tick in ticks)
+ {
+ PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label);
+ largestLabelWidth = Math.Max(largestLabelWidth, size.Width);
+ }
+
+ // ensure axis panels do not get smaller than the largest label
+ plt.Axes.Bottom.MinimumSize = largestLabelWidth;
+ plt.Axes.Right.MinimumSize = largestLabelWidth;
+ }
+
+ var bars = data
+ .Select((d, index) => new Bar()
+ {
+ Position = ticks[index].Position,
+ Value = d.Mean,
+ Error = d.StdError,
+ FillColor = legendPalette[d.JobId]
+ });
+ plt.Add.Bars(bars);
+
+ // Tell the plot to autoscale with no padding beneath the bars
+ plt.Axes.Margins(bottom: 0, right: .2);
+
+ plt.PlottableList.AddRange(annotations);
+
+ plt.SavePng(fileName, this.Width, this.Height);
+ return Path.GetFullPath(fileName);
+ }
+
+ /// <summary>
+ /// Provides a list of annotations to put over the data area.
+ /// </summary>
+ /// <param name="version">The version to be displayed.</param>
+ /// <returns>A list of annotations for every plot.</returns>
+ private IReadOnlyList<Annotation> GetAnnotations(string version)
+ {
+ var versionAnnotation = new Annotation()
+ {
+ Label =
+ {
+ Text = version,
+ FontSize = 14,
+ ForeColor = new Color(0, 0, 0, 100)
+ },
+ OffsetY = 10,
+ OffsetX = 20,
+ Alignment = Alignment.LowerRight
+ };
+
+
+ return new[] { versionAnnotation };
+ }
+ }
+}

tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <Import Project="..\..\build\common.props" />
+ <PropertyGroup>
+ <TargetFrameworks>net8.0;net462</TargetFrameworks>
+ <IsPackable>false</IsPackable>
+ <IsTestProject>true</IsTestProject>
+ <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+ <GenerateDocumentationFile>false</GenerateDocumentationFile>
+ </PropertyGroup>
+ <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
+ <PackageReference Include="Microsoft.NETCore.Platforms" Version="6.0.0" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\..\src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj" />
+ <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+ <ProjectReference Include="..\BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" />
+ </ItemGroup>
+ <ItemGroup>
+ <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+ <PackageReference Include="xunit" Version="2.6.2" />
+ <PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
+ <PrivateAssets>all</PrivateAssets>
+ <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+ </PackageReference>
+ </ItemGroup>
+</Project>