From ad937654174e521741aac620e16635a8ff14b1c9 Mon Sep 17 00:00:00 2001 From: Cameron Aavik <99771732+caaavik-msft@users.noreply.github.com> Date: Thu, 21 Sep 2023 22:52:40 +1000 Subject: [PATCH] Add event processor functionality (#2420) --- .../Configs/ConfigExtensions.cs | 2 + src/BenchmarkDotNet/Configs/DebugConfig.cs | 2 + src/BenchmarkDotNet/Configs/DefaultConfig.cs | 3 + src/BenchmarkDotNet/Configs/IConfig.cs | 2 + .../Configs/ImmutableConfig.cs | 5 + .../Configs/ImmutableConfigBuilder.cs | 2 + src/BenchmarkDotNet/Configs/ManualConfig.cs | 9 + .../CompositeEventProcessor.cs | 97 ++++++ .../EventProcessors/EventProcessor.cs | 25 ++ .../Running/BenchmarkRunnerClean.cs | 60 +++- .../EventProcessorTests.cs | 312 ++++++++++++++++++ 11 files changed, 509 insertions(+), 10 deletions(-) create mode 100644 src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs create mode 100644 src/BenchmarkDotNet/EventProcessors/EventProcessor.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index fa0920c7f9..4618b050b4 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -110,6 +111,7 @@ public static class ConfigExtensions [Obsolete("This method will soon be removed, please start using .AddLogicalGroupRules() instead.")] [EditorBrowsable(EditorBrowsableState.Never)] public static IConfig With(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.AddLogicalGroupRules(rules); [PublicAPI] public static ManualConfig AddLogicalGroupRules(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.With(c => c.AddLogicalGroupRules(rules)); + [PublicAPI] public static ManualConfig AddEventProcessor(this IConfig config, EventProcessor eventProcessor) => config.With(c => c.AddEventProcessor(eventProcessor)); [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames)); [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns)); diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 0d66540bb1..0fdda7b08d 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -61,6 +62,7 @@ public abstract class DebugConfig : IConfig public IEnumerable GetDiagnosers() => Array.Empty(); public IEnumerable GetAnalysers() => Array.Empty(); public IEnumerable GetHardwareCounters() => Array.Empty(); + public IEnumerable GetEventProcessors() => Array.Empty(); public IEnumerable GetFilters() => Array.Empty(); public IEnumerable GetColumnHidingRules() => Array.Empty(); diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 8d1df6285b..2323d4fdc4 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Filters; @@ -108,6 +109,8 @@ public string ArtifactsPath public IEnumerable GetFilters() => Array.Empty(); + public IEnumerable GetEventProcessors() => Array.Empty(); + public IEnumerable GetColumnHidingRules() => Array.Empty(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index ba34cfeddf..b311c235f5 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -27,6 +28,7 @@ public interface IConfig IEnumerable GetHardwareCounters(); IEnumerable GetFilters(); IEnumerable GetLogicalGroupRules(); + IEnumerable GetEventProcessors(); IEnumerable GetColumnHidingRules(); IOrderer? Orderer { get; } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index b5d84eaf56..1216ef3b16 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -31,6 +32,7 @@ public sealed class ImmutableConfig : IConfig private readonly ImmutableHashSet hardwareCounters; private readonly ImmutableHashSet filters; private readonly ImmutableArray rules; + private readonly ImmutableHashSet eventProcessors; private readonly ImmutableArray columnHidingRules; internal ImmutableConfig( @@ -45,6 +47,7 @@ internal ImmutableConfig( ImmutableArray uniqueRules, ImmutableArray uniqueColumnHidingRules, ImmutableHashSet uniqueRunnableJobs, + ImmutableHashSet uniqueEventProcessors, ConfigUnionRule unionRule, string artifactsPath, CultureInfo cultureInfo, @@ -66,6 +69,7 @@ internal ImmutableConfig( rules = uniqueRules; columnHidingRules = uniqueColumnHidingRules; jobs = uniqueRunnableJobs; + eventProcessors = uniqueEventProcessors; UnionRule = unionRule; ArtifactsPath = artifactsPath; CultureInfo = cultureInfo; @@ -96,6 +100,7 @@ internal ImmutableConfig( public IEnumerable GetHardwareCounters() => hardwareCounters; public IEnumerable GetFilters() => filters; public IEnumerable GetLogicalGroupRules() => rules; + public IEnumerable GetEventProcessors() => eventProcessors; public IEnumerable GetColumnHidingRules() => columnHidingRules; public ILogger GetCompositeLogger() => new CompositeLogger(loggers); diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 0540dd1426..d7c0b5eb0f 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -53,6 +53,7 @@ public static ImmutableConfig Create(IConfig source) var uniqueHidingRules = source.GetColumnHidingRules().ToImmutableArray(); var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet(); + var uniqueEventProcessors = source.GetEventProcessors().ToImmutableHashSet(); return new ImmutableConfig( uniqueColumnProviders, @@ -66,6 +67,7 @@ public static ImmutableConfig Create(IConfig source) uniqueRules, uniqueHidingRules, uniqueRunnableJobs, + uniqueEventProcessors, source.UnionRule, source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath, source.CultureInfo, diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 27eca81863..5b3dffc92b 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Filters; @@ -33,6 +34,7 @@ public class ManualConfig : IConfig private readonly HashSet hardwareCounters = new HashSet(); private readonly List filters = new List(); private readonly List logicalGroupRules = new List(); + private readonly List eventProcessors = new List(); private readonly List columnHidingRules = new List(); public IEnumerable GetColumnProviders() => columnProviders; @@ -45,6 +47,7 @@ public class ManualConfig : IConfig public IEnumerable GetHardwareCounters() => hardwareCounters; public IEnumerable GetFilters() => filters; public IEnumerable GetLogicalGroupRules() => logicalGroupRules; + public IEnumerable GetEventProcessors() => eventProcessors; public IEnumerable GetColumnHidingRules() => columnHidingRules; [PublicAPI] public ConfigOptions Options { get; set; } @@ -221,6 +224,12 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule return this; } + public ManualConfig AddEventProcessor(EventProcessor eventProcessor) + { + this.eventProcessors.Add(eventProcessor); + return this; + } + [PublicAPI] public ManualConfig HideColumns(params string[] columnNames) { diff --git a/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs b/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs new file mode 100644 index 0000000000..a73ea54636 --- /dev/null +++ b/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.EventProcessors +{ + internal sealed class CompositeEventProcessor : EventProcessor + { + private readonly HashSet eventProcessors; + + public CompositeEventProcessor(BenchmarkRunInfo[] benchmarkRunInfos) + { + var eventProcessors = new HashSet(); + + foreach (var info in benchmarkRunInfos) + eventProcessors.AddRange(info.Config.GetEventProcessors()); + + this.eventProcessors = eventProcessors; + } + + public override void OnStartValidationStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartValidationStage(); + } + + public override void OnValidationError(ValidationError validationError) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnValidationError(validationError); + } + + public override void OnEndValidationStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndValidationStage(); + } + + public override void OnStartBuildStage(IReadOnlyList partitions) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartBuildStage(partitions); + } + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnBuildComplete(buildPartition, buildResult); + } + + public override void OnEndBuildStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndBuildStage(); + } + + public override void OnStartRunStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartRunStage(); + } + + public override void OnEndRunStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndRunStage(); + } + + public override void OnStartRunBenchmarksInType(Type type, IReadOnlyList benchmarks) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartRunBenchmarksInType(type, benchmarks); + } + + public override void OnEndRunBenchmarksInType(Type type, Summary summary) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndRunBenchmarksInType(type, summary); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndRunBenchmark(benchmarkCase, report); + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartRunBenchmark(benchmarkCase); + } + } +} diff --git a/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs b/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs new file mode 100644 index 0000000000..339b98fa5e --- /dev/null +++ b/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.EventProcessors +{ + public abstract class EventProcessor + { + public virtual void OnStartValidationStage() { } + public virtual void OnValidationError(ValidationError validationError) { } + public virtual void OnEndValidationStage() { } + public virtual void OnStartBuildStage(IReadOnlyList partitions) { } + public virtual void OnBuildComplete(BuildPartition partition, BuildResult buildResult) { } + public virtual void OnEndBuildStage() { } + public virtual void OnStartRunStage() { } + public virtual void OnStartRunBenchmarksInType(Type type, IReadOnlyList benchmarks) { } + public virtual void OnEndRunBenchmarksInType(Type type, Summary summary) { } + public virtual void OnStartRunBenchmark(BenchmarkCase benchmarkCase) { } + public virtual void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) { } + public virtual void OnEndRunStage() { } + } +} diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 80a3ef0897..ae7ad0faf2 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; @@ -54,6 +55,9 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath))) { var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); + var eventProcessor = new CompositeEventProcessor(benchmarkRunInfos); + + eventProcessor.OnStartValidationStage(); compositeLogger.WriteLineInfo("// Validating benchmarks:"); @@ -61,6 +65,9 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) validationErrors.AddRange(Validate(supportedBenchmarks)); + foreach (var validationError in validationErrors) + eventProcessor.OnValidationError(validationError); + PrintValidationErrors(compositeLogger, validationErrors); if (validationErrors.Any(validationError => validationError.IsCritical)) @@ -69,16 +76,24 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; + eventProcessor.OnEndValidationStage(); + int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); int benchmarksToRunCount = totalBenchmarkCount - (idToResume + 1); // ids are indexed from 0 compositeLogger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****"); var globalChronometer = Chronometer.Start(); + var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); - var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer); + eventProcessor.OnStartBuildStage(buildPartitions); + var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer, eventProcessor); + var allBuildsHaveFailed = buildResults.Values.All(buildResult => !buildResult.IsBuildSuccess); + eventProcessor.OnEndBuildStage(); + eventProcessor.OnStartRunStage(); + try { var results = new List(); @@ -102,9 +117,11 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) } } - var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, artifactsToCleanup, + eventProcessor.OnStartRunBenchmarksInType(benchmarkRunInfo.Type, benchmarkRunInfo.BenchmarksCases); + var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, eventProcessor, artifactsToCleanup, resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount, taskbarProgress); + eventProcessor.OnEndRunBenchmarksInType(benchmarkRunInfo.Type, summary); if (!benchmarkRunInfo.Config.Options.IsSet(ConfigOptions.JoinSummary)) PrintSummary(compositeLogger, benchmarkRunInfo.Config, summary); @@ -148,6 +165,8 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) Cleanup(compositeLogger, new HashSet(artifactsToCleanup.Distinct())); compositeLogger.WriteLineInfo("Artifacts cleanup is finished"); compositeLogger.Flush(); + + eventProcessor.OnEndRunStage(); } } } @@ -156,6 +175,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Dictionary buildResults, IResolver resolver, ILogger logger, + EventProcessor eventProcessor, List artifactsToCleanup, string resultsFolderPath, string logFilePath, @@ -198,7 +218,10 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles)) artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); + eventProcessor.OnStartRunBenchmark(benchmark); var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult); + eventProcessor.OnEndRunBenchmark(benchmark, report); + if (report.AllMeasurements.Any(m => m.Operations == 0)) throw new InvalidOperationException("An iteration with 'Operations == 0' detected"); reports.Add(report); @@ -347,7 +370,7 @@ private static ImmutableArray Validate(params BenchmarkRunInfo[ return validationErrors.ToImmutableArray(); } - private static Dictionary BuildInParallel(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer) + private static Dictionary BuildInParallel(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer, EventProcessor eventProcessor) { logger.WriteLineHeader($"// ***** Building {buildPartitions.Length} exe(s) in Parallel: Start *****"); @@ -357,8 +380,18 @@ private static Dictionary BuildInParallel(ILogger l var buildResults = buildPartitions .AsParallel() - .Select(buildPartition => (buildPartition, buildResult: Build(buildPartition, rootArtifactsFolderPath, buildLogger))) - .ToDictionary(result => result.buildPartition, result => result.buildResult); + .Select(buildPartition => (Partition: buildPartition, Result: Build(buildPartition, rootArtifactsFolderPath, buildLogger))) + .AsSequential() // Ensure that build completion events are processed sequentially + .Select(build => + { + // If the generation was successful, but the build was not, we will try building sequentially + // so don't send the OnBuildComplete event yet. + if (buildPartitions.Length <= 1 || !build.Result.IsGenerateSuccess || build.Result.IsBuildSuccess) + eventProcessor.OnBuildComplete(build.Partition, build.Result); + + return build; + }) + .ToDictionary(build => build.Partition, build => build.Result); var afterParallelBuild = globalChronometer.GetElapsed(); @@ -370,8 +403,15 @@ private static Dictionary BuildInParallel(ILogger l logger.WriteLineHeader("// ***** Failed to build in Parallel, switching to sequential build *****"); foreach (var buildPartition in buildPartitions) - if (buildResults[buildPartition].IsGenerateSuccess && !buildResults[buildPartition].IsBuildSuccess && !buildResults[buildPartition].TryToExplainFailureReason(out string _)) - buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, buildLogger); + { + if (buildResults[buildPartition].IsGenerateSuccess && !buildResults[buildPartition].IsBuildSuccess) + { + if (!buildResults[buildPartition].TryToExplainFailureReason(out string _)) + buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, buildLogger); + + eventProcessor.OnBuildComplete(buildPartition, buildResults[buildPartition]); + } + } var afterSequentialBuild = globalChronometer.GetElapsed(); @@ -672,9 +712,9 @@ private static void LogProgress(ILogger logger, in StartedClock runsChronometer, $" Estimated finish {estimatedEnd:yyyy-MM-dd H:mm} ({(int)fromNow.TotalHours}h {fromNow.Minutes}m from now) **"; logger.WriteLineHeader(message); - consoleTitler.UpdateTitle ($"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining - {(int)fromNow.TotalHours}h {fromNow.Minutes}m to finish"); + consoleTitler.UpdateTitle($"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining - {(int)fromNow.TotalHours}h {fromNow.Minutes}m to finish"); - taskbarProgress.SetProgress((float) executedBenchmarkCount / totalBenchmarkCount); + taskbarProgress.SetProgress((float)executedBenchmarkCount / totalBenchmarkCount); } private static TimeSpan GetEstimatedFinishTime(in StartedClock runsChronometer, int benchmarksToRunCount, int executedBenchmarkCount) @@ -729,4 +769,4 @@ private static int GetIdToResume(string rootArtifactsFolderPath, string currentL return -1; } } -} +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs new file mode 100644 index 0000000000..4cede6835b --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -0,0 +1,312 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.EventProcessors; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class EventProcessorTests + { + [Fact] + public void WhenUsingEventProcessorAndNoBenchmarks() + { + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) }); + Assert.Single(events); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + } + + [Fact] + public void WhenUsingEventProcessorOnSingleClass() + { + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }); + + Assert.Equal(13, events.Count); + + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[2].EventType); + Assert.Equal(nameof(EventProcessor.OnBuildComplete), events[3].EventType); + Assert.Equal(nameof(EventProcessor.OnEndBuildStage), events[4].EventType); + Assert.Equal(nameof(EventProcessor.OnStartRunStage), events[5].EventType); + + var benchmarkTypeAndMethods = new List<(Type Type, string[] MethodNames)> + { + (typeof(ClassA), new[]{ nameof(ClassA.Method1), nameof(ClassA.Method2) }) + }; + + int eventIndex = 6; + foreach ((var type, var methodNames) in benchmarkTypeAndMethods) + { + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + + foreach (var method in methodNames) + { + var methodDescriptor = type.GetMethod(method); + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunStage), events[eventIndex].EventType); + } + + [Fact] + public void WhenUsingEventProcessorOnMultipleClasses() + { + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA), typeof(ClassB) }); + + Assert.Equal(23, events.Count); + + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[2].EventType); + Assert.Equal(nameof(EventProcessor.OnBuildComplete), events[3].EventType); + Assert.Equal(nameof(EventProcessor.OnEndBuildStage), events[4].EventType); + Assert.Equal(nameof(EventProcessor.OnStartRunStage), events[5].EventType); + + var benchmarkTypeAndMethods = new List<(Type Type, string[] MethodNames)> + { + (typeof(ClassA), new[]{ nameof(ClassA.Method1), nameof(ClassA.Method2) }), + (typeof(ClassB), new[]{ nameof(ClassB.Method1), nameof(ClassB.Method2), nameof(ClassB.Method3), nameof(ClassB.Method4) }) + }; + + int eventIndex = 6; + foreach ((var type, var methodNames) in benchmarkTypeAndMethods) + { + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + + foreach (var method in methodNames) + { + var methodDescriptor = type.GetMethod(method); + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunStage), events[eventIndex].EventType); + } + + [Fact] + public void WhenUsingEventProcessorWithValidationErrors() + { + var validator = new ErrorAllCasesValidator(); + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, validator); + + Assert.Equal(15, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method1)), (events[1].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[2].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method2)), (events[2].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[3].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[4].EventType); + } + + [Fact] + public void WhenUsingEventProcessorWithUnsupportedBenchmark() + { + var toolchain = new AllUnsupportedToolchain(); + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, toolchain: toolchain); + + Assert.Equal(3, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method1)), (events[1].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[2].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method2)), (events[2].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + } + + [Fact] + public void WhenUsingEventProcessorWithBuildFailures() + { + var toolchain = new Toolchain("Build Failure", new AllFailsGenerator(), null, null); + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, toolchain: toolchain); + + Assert.Equal(9, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[2].EventType); + Assert.Equal(nameof(EventProcessor.OnBuildComplete), events[3].EventType); + Assert.False((events[3].Args[1] as BuildResult).IsGenerateSuccess); + Assert.Equal(nameof(EventProcessor.OnEndBuildStage), events[4].EventType); + Assert.Equal(nameof(EventProcessor.OnStartRunStage), events[5].EventType); + } + + private List RunBenchmarksAndRecordEvents(Type[] types, IValidator? validator = null, IToolchain? toolchain = null) + { + var eventProcessor = new LoggingEventProcessor(); + var job = new Job(Job.Dry); + if (toolchain != null) + job.Infrastructure.Toolchain = toolchain; + + var config = new ManualConfig() + .AddJob(job) + .AddEventProcessor(eventProcessor) + .WithOptions(ConfigOptions.DisableOptimizationsValidator) + .AddExporter(new MockExporter()) // only added to prevent validation warnings about a lack of exporters + .AddLogger(ConsoleLogger.Default) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); + if (validator != null) + config = config.AddValidator(validator); + _ = BenchmarkRunner.Run(types, config); + return eventProcessor.Events; + } + + public class ClassA + { + [Benchmark] + public void Method1() { } + [Benchmark] + public void Method2() { } + } + + public class ClassB + { + [Benchmark] + public void Method1() { } + [Benchmark] + public void Method2() { } + [Benchmark] + public void Method3() { } + [Benchmark] + public void Method4() { } + } + + public class ClassEmpty { } + + public class ErrorAllCasesValidator : IValidator + { + public bool TreatsWarningsAsErrors => true; + + public IEnumerable Validate(ValidationParameters validationParameters) + { + foreach (var benchmark in validationParameters.Benchmarks) + yield return new ValidationError(false, "Mock Validation", benchmark); + } + } + + public class AllUnsupportedToolchain : Toolchain + { + public AllUnsupportedToolchain() : base("AllUnsupported", null, null, null) + { + } + + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) + { + yield return new ValidationError(true, "Unsupported Benchmark", benchmarkCase); + } + } + + public class AllFailsGenerator : IGenerator + { + public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath) + { + return GenerateResult.Failure(ArtifactsPaths.Empty, new List(), new Exception("Generation Failed")); + } + } + + public class LoggingEventProcessor : EventProcessor + { + public class EventData + { + public EventData(string eventType, IReadOnlyList args) + { + EventType = eventType; + Args = args; + } + + public string EventType { get; } + public IReadOnlyList Args { get; } + } + + public List Events { get; } = new List(); + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + Events.Add(new EventData(nameof(OnBuildComplete), new object[] { buildPartition, buildResult })); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + Events.Add(new EventData(nameof(OnEndRunBenchmark), new object[] { benchmarkCase, report })); + } + + public override void OnEndRunBenchmarksInType(Type type, Summary summary) + { + Events.Add(new EventData(nameof(OnEndRunBenchmarksInType), new object[] { type, summary })); + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + Events.Add(new EventData(nameof(OnStartRunBenchmark), new object[] { benchmarkCase })); + } + + public override void OnStartRunBenchmarksInType(Type type, IReadOnlyList benchmarks) + { + Events.Add(new EventData(nameof(OnStartRunBenchmarksInType), new object[] { type, benchmarks })); + } + + public override void OnStartBuildStage(IReadOnlyList partitions) + { + Events.Add(new EventData(nameof(OnStartBuildStage), new object[] { partitions })); + } + + public override void OnStartRunStage() + { + Events.Add(new EventData(nameof(OnStartRunStage), new object[] { })); + } + + public override void OnStartValidationStage() + { + Events.Add(new EventData(nameof(OnStartValidationStage), new object[] { })); + } + + public override void OnValidationError(ValidationError validationError) + { + Events.Add(new EventData(nameof(OnValidationError), new object[] { validationError })); + } + + public override void OnEndValidationStage() + { + Events.Add(new EventData(nameof(OnEndValidationStage), new object[] { })); + } + + public override void OnEndBuildStage() + { + Events.Add(new EventData(nameof(OnEndBuildStage), new object[] { })); + } + + public override void OnEndRunStage() + { + Events.Add(new EventData(nameof(OnEndRunStage), new object[] { })); + } + } + } +}