diff --git a/BenchmarkDotNet.sln b/BenchmarkDotNet.sln index d51cb4e029..1df6c0aabd 100644 --- a/BenchmarkDotNet.sln +++ b/BenchmarkDotNet.sln @@ -55,6 +55,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Diagnostics.dotMemory", "src\BenchmarkDotNet.Diagnostics.dotMemory\BenchmarkDotNet.Diagnostics.dotMemory.csproj", "{2E2283A3-6DA6-4482-8518-99D6D9F689AB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting", "src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj", "{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -149,6 +153,14 @@ Global {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.Build.0 = Debug|Any CPU {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.ActiveCfg = Release|Any CPU {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.Build.0 = Release|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.Build.0 = Release|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -176,6 +188,8 @@ Global {AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9} {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} {2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F} diff --git a/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj new file mode 100644 index 0000000000..799a055885 --- /dev/null +++ b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj @@ -0,0 +1,19 @@ + + + + BenchmarkDotNet plotting export support. + netstandard2.0 + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + + True + enable + + + + + + + + diff --git a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs new file mode 100644 index 0000000000..cfb7a735b7 --- /dev/null +++ b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Properties; +using BenchmarkDotNet.Reports; +using ScottPlot; +using ScottPlot.Plottables; + +namespace BenchmarkDotNet.Exporters.Plotting +{ + /// + /// Provides plot exports as .png files. + /// + public class ScottPlotExporter : IExporter + { + /// + /// Default instance of the exporter with default configuration. + /// + public static readonly IExporter Default = new ScottPlotExporter(); + + /// + /// Gets the name of the Exporter type. + /// + public string Name => nameof(ScottPlotExporter); + + /// + /// Initializes a new instance of ScottPlotExporter. + /// + /// The width of all plots in pixels (optional). Defaults to 1920. + /// The height of all plots in pixels (optional). Defaults to 1080. + public ScottPlotExporter(int width = 1920, int height = 1080) + { + this.Width = width; + this.Height = height; + this.IncludeBarPlot = true; + this.RotateLabels = true; + } + + /// + /// Gets or sets the width of all plots in pixels. + /// + public int Width { get; set; } + + /// + /// Gets or sets the height of all plots in pixels. + /// + public int Height { get; set; } + + /// + /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated. + /// This allows for longer labels at the expense of chart height. + /// + public bool RotateLabels { get; set; } + + /// + /// Gets or sets a value indicating whether a bar plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBarPlot { get; set; } + + /// + /// Not supported. + /// + /// This parameter is not used. + /// This parameter is not used. + /// + public void ExportToLog(Summary summary, ILogger logger) + { + throw new NotSupportedException(); + } + + /// + /// Exports plots to .png file. + /// + /// The summary to be exported. + /// Logger to output to. + /// The file paths of every plot exported. + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + var title = summary.Title; + var version = BenchmarkDotNetInfo.Instance.BrandTitle; + var annotations = GetAnnotations(version); + + var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements)); + + foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name)) + { + var benchmarkName = benchmark.Key; + + // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param]. + var timeStats = from report in benchmark + let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty) + from measurement in report.AllMeasurements + let measurementValue = measurement.Nanoseconds / measurement.Operations + group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g + select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList())); + + if (this.IncludeBarPlot) + { + // -barplot.png + yield return CreateBarPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + /* TODO: Rest of the RPlotExporter plots. + -boxplot.png + --density.png + --facetTimeline.png + --facetTimelineSmooth.png + ---timelineSmooth.png + ---timelineSmooth.png*/ + } + } + + /// + /// Calculate Standard Deviation. + /// + /// Values to calculate from. + /// Standard deviation of values. + private static double StandardError(IReadOnlyList values) + { + double average = values.Average(); + double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum(); + double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count); + return standardDeviation / Math.Sqrt(values.Count); + } + + /// + /// Gets the lowest appropriate time scale across all measurements. + /// + /// All measurements + /// A unit and scaling factor to convert from nanoseconds. + private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable values) + { + var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min(); + if (minValue > 1000000000d) + { + return ("sec", 1000000000d); + } + + if (minValue > 1000000d) + { + return ("ms", 1000000d); + } + + if (minValue > 1000d) + { + return ("us", 1000d); + } + + return ("ns", 1d); + } + + private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, 28); + plt.YLabel(yLabel); + plt.XLabel(xLabel); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Location = Alignment.UpperRight; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + Label = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label); + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + var bars = data + .Select((d, index) => new Bar() + { + Position = ticks[index].Position, + Value = d.Mean, + Error = d.StdError, + FillColor = legendPalette[d.JobId] + }); + plt.Add.Bars(bars); + + // Tell the plot to autoscale with no padding beneath the bars + plt.Axes.Margins(bottom: 0, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + /// + /// Provides a list of annotations to put over the data area. + /// + /// The version to be displayed. + /// A list of annotations for every plot. + private IReadOnlyList GetAnnotations(string version) + { + var versionAnnotation = new Annotation() + { + Label = + { + Text = version, + FontSize = 14, + ForeColor = new Color(0, 0, 0, 100) + }, + OffsetY = 10, + OffsetX = 20, + Alignment = Alignment.LowerRight + }; + + + return new[] { versionAnnotation }; + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj new file mode 100644 index 0000000000..0c61fd72af --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj @@ -0,0 +1,26 @@ + + + + net8.0;net462 + false + true + true + false + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs new file mode 100644 index 0000000000..a585b9df86 --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs @@ -0,0 +1,240 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Tests.Mocks; +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Exporters.Plotting.Tests +{ + public class ScottPlotExporterTests + { + public static TheoryData GetGroupBenchmarkTypes() + { + var data = new TheoryData(); + foreach (var type in typeof(BaselinesBenchmarks).GetNestedTypes()) + data.Add(type); + return data; + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BarPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter(); + var summary = MockFactory.CreateSummary(benchmarkType); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + } + + [SuppressMessage("ReSharper", "InconsistentNaming")] + public static class BaselinesBenchmarks + { + /* NoBaseline */ + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class NoBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByMethod)] + public class NoBaseline_MethodsParamsJobs_GroupByMethod + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByJob)] + public class NoBaseline_MethodsParamsJobs_GroupByJob + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)] + public class NoBaseline_MethodsParamsJobs_GroupByParams + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByCategory + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy( + BenchmarkLogicalGroupRule.ByMethod, + BenchmarkLogicalGroupRule.ByJob, + BenchmarkLogicalGroupRule.ByParams, + BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByAll + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + /* MethodBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_Methods + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_MethodsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* JobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsJobs + { + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* MethodJobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Invalid */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class Invalid_TwoMethodBaselines + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark(Baseline = true)] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] + public class Invalid_TwoJobBaselines + { + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Escape Params */ + + public class Escape_ParamsAndArguments + { + [Params("\t", "\n")] public string StringParam; + + [Arguments('\t')] + [Arguments('\n')] + [Benchmark] public void Foo(char charArg) { } + [Benchmark] public void Bar() { } + } + } + } +} \ No newline at end of file