Feature: Box Plot support for ScottPlotExporter

src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Properties;
 using BenchmarkDotNet.Reports;
@@ -35,6 +36,7 @@ public ScottPlotExporter(int width = 1920, int height = 1080)
  this.Width = width;
  this.Height = height;
  this.IncludeBarPlot = true;
+ this.IncludeBoxPlot = true;
  this.RotateLabels = true;
  }
 
@@ -48,6 +50,16 @@ public ScottPlotExporter(int width = 1920, int height = 1080)
  /// </summary>
  public int Height { get; set; }
 
+ /// <summary>
+ /// Gets or sets the common font size for ticks, labels etc. (defaults to 14).
+ /// </summary>
+ public int FontSize { get; set; } = 14;
+
+ /// <summary>
+ /// Gets or sets the font size for the chart title. (defaults to 28).
+ /// </summary>
+ public int TitleFontSize { get; set; } = 28;
+
  /// <summary>
  /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated.
  /// This allows for longer labels at the expense of chart height.
@@ -60,6 +72,12 @@ public ScottPlotExporter(int width = 1920, int height = 1080)
  /// </summary>
  public bool IncludeBarPlot { get; set; }
 
+ /// <summary>
+ /// Gets or sets a value indicating whether a box plot or whisker plot for time-per-op
+ /// measurement values should be exported.
+ /// </summary>
+ public bool IncludeBoxPlot { get; set; }
+
  /// <summary>
  /// Not supported.
  /// </summary>
@@ -83,7 +101,8 @@ public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
  var version = BenchmarkDotNetInfo.Instance.BrandTitle;
  var annotations = GetAnnotations(version);
 
- var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements));
+ var (timeUnit, timeScale) = GetTimeUnit(summary.Reports
+ .SelectMany(m => m.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Result))));
 
  foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name))
  {
@@ -93,9 +112,10 @@ public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
  var timeStats = from report in benchmark
  let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty)
  from measurement in report.AllMeasurements
+ where measurement.Is(IterationMode.Workload, IterationStage.Result)
  let measurementValue = measurement.Nanoseconds / measurement.Operations
  group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g
- select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList()));
+ select new ChartStats(g.Key.Target, g.Key.JobId, g.ToList());
 
  if (this.IncludeBarPlot)
  {
@@ -109,8 +129,19 @@ from measurement in report.AllMeasurements
  annotations);
  }
 
+ if (this.IncludeBoxPlot)
+ {
+ // <BenchmarkName>-boxplot.png
+ yield return CreateBoxPlot(
+ $"{title} - {benchmarkName}",
+ Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-boxplot.png"),
+ $"Time ({timeUnit})",
+ "Target",
+ timeStats,
+ annotations);
+ }
+
  /* TODO: Rest of the RPlotExporter plots.
- <BenchmarkName>-boxplot.png
  <BenchmarkName>-<MethodName>-density.png
  <BenchmarkName>-<MethodName>-facetTimeline.png
  <BenchmarkName>-<MethodName>-facetTimelineSmooth.png
@@ -158,12 +189,12 @@ private static double StandardError(IReadOnlyList<double> values)
  return ("ns", 1d);
  }
 
- private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList<Annotation> annotations)
+ private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<ChartStats> data, IReadOnlyList<Annotation> annotations)
  {
  Plot plt = new Plot();
- plt.Title(title, 28);
- plt.YLabel(yLabel);
- plt.XLabel(xLabel);
+ plt.Title(title, this.TitleFontSize);
+ plt.YLabel(yLabel, this.FontSize);
+ plt.XLabel(xLabel, this.FontSize);
 
  var palette = new ScottPlot.Palettes.Category10();
 
@@ -174,6 +205,7 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
 
  plt.Legend.IsVisible = true;
  plt.Legend.Location = Alignment.UpperRight;
+ plt.Legend.Font.Size = this.FontSize;
  var legend = data.Select(d => d.JobId)
  .Distinct()
  .Select((label, index) => new LegendItem()
@@ -189,8 +221,11 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
  var ticks = data
  .Select((d, index) => new Tick(index, d.Target))
  .ToArray();
+
+ plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize;
  plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
  plt.Axes.Bottom.MajorTickStyle.Length = 0;
+ plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize;
 
  if (this.RotateLabels)
  {
@@ -206,7 +241,7 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
  }
 
  // ensure axis panels do not get smaller than the largest label
- plt.Axes.Bottom.MinimumSize = largestLabelWidth;
+ plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2;
  plt.Axes.Right.MinimumSize = largestLabelWidth;
  }
 
@@ -229,6 +264,89 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
  return Path.GetFullPath(fileName);
  }
 
+ private string CreateBoxPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<ChartStats> data, IReadOnlyList<Annotation> annotations)
+ {
+ Plot plt = new Plot();
+ plt.Title(title, this.TitleFontSize);
+ plt.YLabel(yLabel, this.FontSize);
+ plt.XLabel(xLabel, this.FontSize);
+
+ var palette = new ScottPlot.Palettes.Category10();
+
+ var legendPalette = data.Select(d => d.JobId)
+ .Distinct()
+ .Select((jobId, index) => (jobId, index))
+ .ToDictionary(t => t.jobId, t => palette.GetColor(t.index));
+
+ plt.Legend.IsVisible = true;
+ plt.Legend.Location = Alignment.UpperRight;
+ plt.Legend.Font.Size = this.FontSize;
+ var legend = data.Select(d => d.JobId)
+ .Distinct()
+ .Select((label, index) => new LegendItem()
+ {
+ Label = label,
+ FillColor = legendPalette[label]
+ })
+ .ToList();
+
+ plt.Legend.ManualItems.AddRange(legend);
+
+ var jobCount = plt.Legend.ManualItems.Count;
+ var ticks = data
+ .Select((d, index) => new Tick(index, d.Target))
+ .ToArray();
+
+ plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize;
+ plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
+ plt.Axes.Bottom.MajorTickStyle.Length = 0;
+ plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize;
+
+ if (this.RotateLabels)
+ {
+ plt.Axes.Bottom.TickLabelStyle.Rotation = 45;
+ plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft;
+
+ // determine the width of the largest tick label
+ float largestLabelWidth = 0;
+ foreach (Tick tick in ticks)
+ {
+ PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label);
+ largestLabelWidth = Math.Max(largestLabelWidth, size.Width);
+ }
+
+ // ensure axis panels do not get smaller than the largest label
+ plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2;
+ plt.Axes.Right.MinimumSize = largestLabelWidth;
+ }
+
+ int globalIndex = 0;
+ foreach (var (targetGroup, targetGroupIndex) in data.GroupBy(s => s.Target).Select((targetGroup, index) => (targetGroup, index)))
+ {
+ var boxes = targetGroup.Select(job => (job.JobId, Stats: job.CalculateBoxPlotStatistics())).Select((j, jobIndex) => new Box()
+ {
+ Position = ticks[globalIndex++].Position,
+ Fill = new FillStyle() { Color = legendPalette[j.JobId] },
+ Stroke = new LineStyle() { Color = Colors.Black },
+ BoxMin = j.Stats.Q1,
+ BoxMax = j.Stats.Q3,
+ WhiskerMin = j.Stats.Min,
+ WhiskerMax = j.Stats.Max,
+ BoxMiddle = j.Stats.Median
+ })
+ .ToList();
+ plt.Add.Boxes(boxes);
+ }
+
+ // Tell the plot to autoscale with a small padding below the boxes.
+ plt.Axes.Margins(bottom: 0.05, right: .2);
+
+ plt.PlottableList.AddRange(annotations);
+
+ plt.SavePng(fileName, this.Width, this.Height);
+ return Path.GetFullPath(fileName);
+ }
+
  /// <summary>
  /// Provides a list of annotations to put over the data area.
  /// </summary>
@@ -252,5 +370,69 @@ private IReadOnlyList<Annotation> GetAnnotations(string version)
 
  return new[] { versionAnnotation };
  }
+
+ private class ChartStats
+ {
+ public ChartStats(string Target, string JobId, IReadOnlyList<double> Values)
+ {
+ this.Target = Target;
+ this.JobId = JobId;
+ this.Values = Values;
+ }
+
+ public string Target { get; }
+
+ public string JobId { get; }
+
+ public IReadOnlyList<double> Values { get; }
+
+ public double Min => this.Values.DefaultIfEmpty(0d).Min();
+
+ public double Max => this.Values.DefaultIfEmpty(0d).Max();
+
+ public double Mean => this.Values.DefaultIfEmpty(0d).Average();
+
+ public double StdError => StandardError(this.Values);
+
+
+ private static (int MidPoint, double Median) CalculateMedian(ReadOnlySpan<double> values)
+ {
+ int n = values.Length;
+ var midPoint = n / 2;
+
+ // Check if count is even, if so use average of the two middle values,
+ // otherwise take the middle value.
+ var median = n % 2 == 0 ? (values[midPoint - 1] + values[midPoint]) / 2d : values[midPoint];
+ return (midPoint, median);
+ }
+
+ /// <summary>
+ /// Calculate the mid points.
+ /// </summary>
+ /// <returns></returns>
+ public (double Min, double Q1, double Median, double Q3, double Max, double[] Outliers) CalculateBoxPlotStatistics()
+ {
+ var values = this.Values.ToArray();
+ Array.Sort(values);
+ var s = values.AsSpan();
+ var (midPoint, median) = CalculateMedian(s);
+
+ var (q1Index, q1) = midPoint > 0 ? CalculateMedian(s.Slice(0, midPoint)) : (midPoint, median);
+ var (q3Index, q3) = midPoint + 1 < s.Length ? CalculateMedian(s.Slice(midPoint + 1)) : (midPoint, median);
+ var iqr = q3 - q1;
+ var lowerFence = q1 - 1.5d * iqr;
+ var upperFence = q3 + 1.5d * iqr;
+ var outliers = values.Where(v => v < lowerFence || v > upperFence).ToArray();
+ var nonOutliers = values.Where(v => v >= lowerFence && v <= upperFence).ToArray();
+ return (
+ nonOutliers.FirstOrDefault(),
+ q1,
+ median,
+ q3,
+ nonOutliers.LastOrDefault(),
+ outliers
+ );
+ }
+ }
  }
 }

tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs

@@ -1,6 +1,9 @@
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Tests.Builders;
 using BenchmarkDotNet.Tests.Mocks;
 using System;
 using System.Diagnostics.CodeAnalysis;
@@ -28,7 +31,11 @@ public void BarPlots(Type benchmarkType)
  var logger = new AccumulationLogger();
  logger.WriteLine("=== " + benchmarkType.Name + " ===");
 
- var exporter = new ScottPlotExporter();
+ var exporter = new ScottPlotExporter()
+ {
+ IncludeBarPlot = true,
+ IncludeBoxPlot = false,
+ };
  var summary = MockFactory.CreateSummary(benchmarkType);
  var filePaths = exporter.ExportToFiles(summary, logger).ToList();
  Assert.NotEmpty(filePaths);
@@ -39,6 +46,50 @@ public void BarPlots(Type benchmarkType)
  output.WriteLine(logger.GetLog());
  }
 
+ [Theory]
+ [MemberData(nameof(GetGroupBenchmarkTypes))]
+ public void BoxPlots(Type benchmarkType)
+ {
+ var logger = new AccumulationLogger();
+ logger.WriteLine("=== " + benchmarkType.Name + " ===");
+
+ var exporter = new ScottPlotExporter()
+ {
+ IncludeBarPlot = false,
+ IncludeBoxPlot = true,
+ };
+ var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 9);
+ var filePaths = exporter.ExportToFiles(summary, logger).ToList();
+ Assert.NotEmpty(filePaths);
+ Assert.All(filePaths, f => File.Exists(f));
+
+ foreach (string filePath in filePaths)
+ logger.WriteLine($"* {filePath}");
+ output.WriteLine(logger.GetLog());
+ }
+
+ [Theory]
+ [MemberData(nameof(GetGroupBenchmarkTypes))]
+ public void BoxPlotsWithOneMeasurement(Type benchmarkType)
+ {
+ var logger = new AccumulationLogger();
+ logger.WriteLine("=== " + benchmarkType.Name + " ===");
+
+ var exporter = new ScottPlotExporter()
+ {
+ IncludeBarPlot = false,
+ IncludeBoxPlot = true,
+ };
+ var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 1);
+ var filePaths = exporter.ExportToFiles(summary, logger).ToList();
+ Assert.NotEmpty(filePaths);
+ Assert.All(filePaths, f => File.Exists(f));
+
+ foreach (string filePath in filePaths)
+ logger.WriteLine($"* {filePath}");
+ output.WriteLine(logger.GetLog());
+ }
+
  [SuppressMessage("ReSharper", "InconsistentNaming")]
  public static class BaselinesBenchmarks
  {