diff --git a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs index cfb7a735b7..dba73d5f8d 100644 --- a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs +++ b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Reports; @@ -35,6 +36,7 @@ public ScottPlotExporter(int width = 1920, int height = 1080) this.Width = width; this.Height = height; this.IncludeBarPlot = true; + this.IncludeBoxPlot = true; this.RotateLabels = true; } @@ -48,6 +50,16 @@ public ScottPlotExporter(int width = 1920, int height = 1080) /// public int Height { get; set; } + /// + /// Gets or sets the common font size for ticks, labels etc. (defaults to 14). + /// + public int FontSize { get; set; } = 14; + + /// + /// Gets or sets the font size for the chart title. (defaults to 28). + /// + public int TitleFontSize { get; set; } = 28; + /// /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated. /// This allows for longer labels at the expense of chart height. @@ -60,6 +72,12 @@ public ScottPlotExporter(int width = 1920, int height = 1080) /// public bool IncludeBarPlot { get; set; } + /// + /// Gets or sets a value indicating whether a box plot or whisker plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBoxPlot { get; set; } + /// /// Not supported. /// @@ -83,7 +101,8 @@ public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) var version = BenchmarkDotNetInfo.Instance.BrandTitle; var annotations = GetAnnotations(version); - var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements)); + var (timeUnit, timeScale) = GetTimeUnit(summary.Reports + .SelectMany(m => m.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Result)))); foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name)) { @@ -93,9 +112,10 @@ public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) var timeStats = from report in benchmark let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty) from measurement in report.AllMeasurements + where measurement.Is(IterationMode.Workload, IterationStage.Result) let measurementValue = measurement.Nanoseconds / measurement.Operations group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g - select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList())); + select new ChartStats(g.Key.Target, g.Key.JobId, g.ToList()); if (this.IncludeBarPlot) { @@ -109,8 +129,19 @@ from measurement in report.AllMeasurements annotations); } + if (this.IncludeBoxPlot) + { + // -boxplot.png + yield return CreateBoxPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-boxplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + /* TODO: Rest of the RPlotExporter plots. - -boxplot.png --density.png --facetTimeline.png --facetTimelineSmooth.png @@ -158,12 +189,12 @@ private static double StandardError(IReadOnlyList values) return ("ns", 1d); } - private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList annotations) + private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) { Plot plt = new Plot(); - plt.Title(title, 28); - plt.YLabel(yLabel); - plt.XLabel(xLabel); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); var palette = new ScottPlot.Palettes.Category10(); @@ -174,6 +205,7 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin plt.Legend.IsVisible = true; plt.Legend.Location = Alignment.UpperRight; + plt.Legend.Font.Size = this.FontSize; var legend = data.Select(d => d.JobId) .Distinct() .Select((label, index) => new LegendItem() @@ -189,8 +221,11 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin var ticks = data .Select((d, index) => new Tick(index, d.Target)) .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; if (this.RotateLabels) { @@ -206,7 +241,7 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin } // ensure axis panels do not get smaller than the largest label - plt.Axes.Bottom.MinimumSize = largestLabelWidth; + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; plt.Axes.Right.MinimumSize = largestLabelWidth; } @@ -229,6 +264,89 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin return Path.GetFullPath(fileName); } + private string CreateBoxPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Location = Alignment.UpperRight; + plt.Legend.Font.Size = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + Label = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label); + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + int globalIndex = 0; + foreach (var (targetGroup, targetGroupIndex) in data.GroupBy(s => s.Target).Select((targetGroup, index) => (targetGroup, index))) + { + var boxes = targetGroup.Select(job => (job.JobId, Stats: job.CalculateBoxPlotStatistics())).Select((j, jobIndex) => new Box() + { + Position = ticks[globalIndex++].Position, + Fill = new FillStyle() { Color = legendPalette[j.JobId] }, + Stroke = new LineStyle() { Color = Colors.Black }, + BoxMin = j.Stats.Q1, + BoxMax = j.Stats.Q3, + WhiskerMin = j.Stats.Min, + WhiskerMax = j.Stats.Max, + BoxMiddle = j.Stats.Median + }) + .ToList(); + plt.Add.Boxes(boxes); + } + + // Tell the plot to autoscale with a small padding below the boxes. + plt.Axes.Margins(bottom: 0.05, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + /// /// Provides a list of annotations to put over the data area. /// @@ -252,5 +370,69 @@ private IReadOnlyList GetAnnotations(string version) return new[] { versionAnnotation }; } + + private class ChartStats + { + public ChartStats(string Target, string JobId, IReadOnlyList Values) + { + this.Target = Target; + this.JobId = JobId; + this.Values = Values; + } + + public string Target { get; } + + public string JobId { get; } + + public IReadOnlyList Values { get; } + + public double Min => this.Values.DefaultIfEmpty(0d).Min(); + + public double Max => this.Values.DefaultIfEmpty(0d).Max(); + + public double Mean => this.Values.DefaultIfEmpty(0d).Average(); + + public double StdError => StandardError(this.Values); + + + private static (int MidPoint, double Median) CalculateMedian(ReadOnlySpan values) + { + int n = values.Length; + var midPoint = n / 2; + + // Check if count is even, if so use average of the two middle values, + // otherwise take the middle value. + var median = n % 2 == 0 ? (values[midPoint - 1] + values[midPoint]) / 2d : values[midPoint]; + return (midPoint, median); + } + + /// + /// Calculate the mid points. + /// + /// + public (double Min, double Q1, double Median, double Q3, double Max, double[] Outliers) CalculateBoxPlotStatistics() + { + var values = this.Values.ToArray(); + Array.Sort(values); + var s = values.AsSpan(); + var (midPoint, median) = CalculateMedian(s); + + var (q1Index, q1) = midPoint > 0 ? CalculateMedian(s.Slice(0, midPoint)) : (midPoint, median); + var (q3Index, q3) = midPoint + 1 < s.Length ? CalculateMedian(s.Slice(midPoint + 1)) : (midPoint, median); + var iqr = q3 - q1; + var lowerFence = q1 - 1.5d * iqr; + var upperFence = q3 + 1.5d * iqr; + var outliers = values.Where(v => v < lowerFence || v > upperFence).ToArray(); + var nonOutliers = values.Where(v => v >= lowerFence && v <= upperFence).ToArray(); + return ( + nonOutliers.FirstOrDefault(), + q1, + median, + q3, + nonOutliers.LastOrDefault(), + outliers + ); + } + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs index 32f69061f6..e717b6b84d 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs @@ -1,6 +1,9 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Builders; using BenchmarkDotNet.Tests.Mocks; using System; using System.Diagnostics.CodeAnalysis; @@ -28,7 +31,11 @@ public void BarPlots(Type benchmarkType) var logger = new AccumulationLogger(); logger.WriteLine("=== " + benchmarkType.Name + " ==="); - var exporter = new ScottPlotExporter(); + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = true, + IncludeBoxPlot = false, + }; var summary = MockFactory.CreateSummary(benchmarkType); var filePaths = exporter.ExportToFiles(summary, logger).ToList(); Assert.NotEmpty(filePaths); @@ -39,6 +46,50 @@ public void BarPlots(Type benchmarkType) output.WriteLine(logger.GetLog()); } + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 9); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlotsWithOneMeasurement(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 1); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + [SuppressMessage("ReSharper", "InconsistentNaming")] public static class BaselinesBenchmarks { diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 8c0ce0019e..edbaba0e3a 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -37,6 +37,27 @@ public static Summary CreateSummary(Type benchmarkType, params IColumnHidingRule TestCultureInfo.Instance, ImmutableArray.Empty, ImmutableArray.Create(columHidingRules)); + } + + public static Summary CreateSummaryWithBiasedDistribution(Type benchmarkType, int min, int median, int max, int n) + { + var runInfo = BenchmarkConverter.TypeToBenchmarks(benchmarkType); + return new Summary( + $"MockSummary-N{n}", + runInfo.BenchmarksCases.Select((benchmark, index) => CreateReportWithBiasedDistribution( + benchmark, + (index + 1) * min, + (index + 1) * median, + (index + 1) * max, + n, + Array.Empty())).ToImmutableArray(), + new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), + string.Empty, + string.Empty, + TimeSpan.FromMinutes(1), + TestCultureInfo.Instance, + ImmutableArray.Empty, + ImmutableArray.Empty); } public static Summary CreateSummary(IConfig config) => new Summary( @@ -120,6 +141,40 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, bool hu return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); } + private static BenchmarkReport CreateReportWithBiasedDistribution(BenchmarkCase benchmarkCase, int min, int median, int max, int n, Metric[] metrics) + { + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); + bool isFoo = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Foo"; + bool isBar = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Bar"; + var measurements = from i in Enumerable.Range(0, Math.Max(1, n / 9)) + from m in isFoo ? new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 8, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } : new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } + select m; + var executeResult = new ExecuteResult(measurements.Take(n).ToList(), default, default, 0); + return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); + } + [LongRunJob] public class MockBenchmarkClass {