Add Event Handler support

src/BenchmarkDotNet/Configs/DebugConfig.cs

@@ -12,6 +12,7 @@
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
 using BenchmarkDotNet.Validators;
 
@@ -62,8 +63,8 @@ public abstract class DebugConfig : IConfig
         public IEnumerable<IDiagnoser> GetDiagnosers() => Array.Empty<IDiagnoser>();
         public IEnumerable<IAnalyser> GetAnalysers() => Array.Empty<IAnalyser>();
         public IEnumerable<HardwareCounter> GetHardwareCounters() => Array.Empty<HardwareCounter>();
+        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => Array.Empty<IBenchmarkEventHandler>();
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
-
         public IOrderer Orderer => DefaultOrderer.Instance;
         public SummaryStyle SummaryStyle => SummaryStyle.Default;
         public ConfigUnionRule UnionRule => ConfigUnionRule.Union;

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -13,6 +13,7 @@
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 
 namespace BenchmarkDotNet.Configs
@@ -103,5 +104,7 @@ public string ArtifactsPath
         public IEnumerable<HardwareCounter> GetHardwareCounters() => Array.Empty<HardwareCounter>();
 
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
+
+        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => Array.Empty<IBenchmarkEventHandler>();
     }
 }

src/BenchmarkDotNet/Configs/IConfig.cs

@@ -10,6 +10,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 using JetBrains.Annotations;
 
@@ -27,6 +28,7 @@ public interface IConfig
         IEnumerable<HardwareCounter> GetHardwareCounters();
         IEnumerable<IFilter> GetFilters();
         IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules();
+        IEnumerable<IBenchmarkEventHandler> GetEventHandlers();
 
         [CanBeNull] IOrderer Orderer { get; }
 

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -32,6 +32,7 @@ public sealed class ImmutableConfig : IConfig
         private readonly ImmutableHashSet<HardwareCounter> hardwareCounters;
         private readonly ImmutableHashSet<IFilter> filters;
         private readonly ImmutableArray<BenchmarkLogicalGroupRule> rules;
+        private readonly ImmutableHashSet<IBenchmarkEventHandler> eventHandlers;
 
         internal ImmutableConfig(
             ImmutableArray<IColumnProvider> uniqueColumnProviders,
@@ -44,6 +45,7 @@ internal ImmutableConfig(
             ImmutableHashSet<IFilter> uniqueFilters,
             ImmutableArray<BenchmarkLogicalGroupRule> uniqueRules,
             ImmutableHashSet<Job> uniqueRunnableJobs,
+            ImmutableHashSet<IBenchmarkEventHandler> uniqueEventHandlers,
             ConfigUnionRule unionRule,
             string artifactsPath,
             CultureInfo cultureInfo,
@@ -63,6 +65,7 @@ internal ImmutableConfig(
             filters = uniqueFilters;
             rules = uniqueRules;
             jobs = uniqueRunnableJobs;
+            eventHandlers = uniqueEventHandlers;
             UnionRule = unionRule;
             ArtifactsPath = artifactsPath;
             CultureInfo = cultureInfo;
@@ -91,12 +94,14 @@ internal ImmutableConfig(
         public IEnumerable<HardwareCounter> GetHardwareCounters() => hardwareCounters;
         public IEnumerable<IFilter> GetFilters() => filters;
         public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => rules;
+        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => eventHandlers;
 
         public ILogger GetCompositeLogger() => new CompositeLogger(loggers);
         public IExporter GetCompositeExporter() => new CompositeExporter(exporters);
         public IValidator GetCompositeValidator() => new CompositeValidator(validators);
         public IAnalyser GetCompositeAnalyser() => new CompositeAnalyser(analysers);
         public IDiagnoser GetCompositeDiagnoser() => new CompositeDiagnoser(diagnosers);
+        public IBenchmarkEventHandler GetCompositeEventHandler() => new CompositeBenchmarkEventHandler(eventHandlers);
 
         public bool HasMemoryDiagnoser() => diagnosers.OfType<MemoryDiagnoser>().Any();
 

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -49,6 +49,7 @@ public static ImmutableConfig Create(IConfig source)
             var uniqueRules = source.GetLogicalGroupRules().ToImmutableArray();
 
             var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet();
+            var uniqueEventHandlers = source.GetEventHandlers().ToImmutableHashSet();
 
             return new ImmutableConfig(
                 uniqueColumnProviders,
@@ -61,6 +62,7 @@ public static ImmutableConfig Create(IConfig source)
                 uniqueFilters,
                 uniqueRules,
                 uniqueRunnableJobs,
+                uniqueEventHandlers,
                 source.UnionRule,
                 source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath,
                 source.CultureInfo,

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -13,6 +13,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 using JetBrains.Annotations;
 
@@ -30,6 +31,7 @@ public class ManualConfig : IConfig
         private readonly HashSet<HardwareCounter> hardwareCounters = new HashSet<HardwareCounter>();
         private readonly List<IFilter> filters = new List<IFilter>();
         private readonly List<BenchmarkLogicalGroupRule> logicalGroupRules = new List<BenchmarkLogicalGroupRule>();
+        private readonly List<IBenchmarkEventHandler> eventHandlers = new List<IBenchmarkEventHandler>();
         private readonly static Conclusion[] emptyConclusion = Array.Empty<Conclusion>();
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
@@ -42,6 +44,7 @@ public class ManualConfig : IConfig
         public IEnumerable<HardwareCounter> GetHardwareCounters() => hardwareCounters;
         public IEnumerable<IFilter> GetFilters() => filters;
         public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => logicalGroupRules;
+        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => eventHandlers;
 
         [PublicAPI] public ConfigOptions Options { get; set; }
         [PublicAPI] public ConfigUnionRule UnionRule { get; set; } = ConfigUnionRule.Union;
@@ -210,6 +213,12 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule
             return this;
         }
 
+        public ManualConfig AddEventHandler(params IBenchmarkEventHandler[] eventHandlers)
+        {
+            this.eventHandlers.AddRange(eventHandlers);
+            return this;
+        }
+
         [PublicAPI]
         public void Add(IConfig config)
         {

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -44,6 +44,9 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
             using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath)))
             {
                 var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger);
+                var eventHandler = CreateCompositeEventHandler(benchmarkRunInfos);
+
+                eventHandler.HandleStartValidationStage();
 
                 var supportedBenchmarks = GetSupportedBenchmarks(benchmarkRunInfos, compositeLogger, resolver);
                 if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any()))
@@ -59,10 +62,14 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
                 compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****");
                 var globalChronometer = Chronometer.Start();
 
+                eventHandler.HandleStartBuildStage();
+
                 var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver);
                 var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer);
                 var allBuildsHaveFailed = buildResults.Values.All(buildResult => !buildResult.IsBuildSuccess);
 
+                eventHandler.HandleStartRunStage();
+
                 try
                 {
                     var results = new List<Summary>();
@@ -76,8 +83,11 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
 
                     foreach (var benchmarkRunInfo in supportedBenchmarks) // we run them in the old order now using the new build artifacts
                     {
+                        var runInfoEventHandler = benchmarkRunInfo.Config.GetCompositeEventHandler();
+                        runInfoEventHandler.HandleRunBenchmarksInType(benchmarkRunInfo.Type, benchmarkRunInfo.BenchmarksCases);
                         var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, artifactsToCleanup,
                             resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount);
+                        runInfoEventHandler.HandleCompletedBenchmarksInType(benchmarkRunInfo.Type, summary);
 
                         if (!benchmarkRunInfo.Config.Options.IsSet(ConfigOptions.JoinSummary))
                             PrintSummary(compositeLogger, benchmarkRunInfo.Config, summary);
@@ -149,6 +159,8 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo,
                 logger.WriteLineInfo($"//   {benchmark.DisplayInfo}");
             logger.WriteLine();
 
+            var eventHandler = config.GetCompositeEventHandler();
+
             using (var powerManagementApplier = new PowerManagementApplier(logger))
             {
                 bool stop = false;
@@ -168,7 +180,10 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo,
                         if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles))
                             artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup);
 
+                        eventHandler.HandleRunBenchmark(benchmark);
                         var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult);
+                        eventHandler.HandleCompletedBenchmark(benchmark, report);
+
                         if (report.AllMeasurements.Any(m => m.Operations == 0))
                             throw new InvalidOperationException("An iteration with 'Operations == 0' detected");
                         reports.Add(report);
@@ -302,15 +317,29 @@ private static ImmutableArray<ValidationError> Validate(BenchmarkRunInfo[] bench
             {
                 var joinedCases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray();
 
-                validationErrors.AddRange(
-                    ConfigCompatibilityValidator
-                        .FailOnError
-                        .Validate(new ValidationParameters(joinedCases, null))
-                    );
+                var compatibilityValidationErrors = ConfigCompatibilityValidator
+                    .FailOnError
+                    .Validate(new ValidationParameters(joinedCases, null));
+
+                foreach (var error in compatibilityValidationErrors)
+                {
+                    validationErrors.Add(error);
+                    foreach (var eventHandler in benchmarks.SelectMany(b => b.Config.GetEventHandlers()).Distinct())
+                        eventHandler.HandleValidationError(error);
+                }
             }
 
             foreach (var benchmarkRunInfo in benchmarks)
-                validationErrors.AddRange(benchmarkRunInfo.Config.GetCompositeValidator().Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, benchmarkRunInfo.Config)));
+            {
+                var config = benchmarkRunInfo.Config;
+                var validator = config.GetCompositeValidator();
+                var eventHandler = config.GetCompositeEventHandler();
+                foreach (var error in validator.Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, config)))
+                {
+                    validationErrors.Add(error);
+                    eventHandler.HandleValidationError(error);
+                }
+            }
 
             foreach (var validationError in validationErrors.Distinct())
                 logger.WriteLineError(validationError.Message);
@@ -348,6 +377,16 @@ private static Dictionary<BuildPartition, BuildResult> BuildInParallel(ILogger l
 
             logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(afterParallelBuild, afterSequentialBuild)}   *****");
 
+            foreach (var buildPartition in buildPartitions)
+            {
+                var buildResult = buildResults[buildPartition];
+                if (!buildResult.IsBuildSuccess)
+                {
+                    foreach (var benchmark in buildPartition.Benchmarks)
+                        benchmark.Config.GetCompositeEventHandler().HandleBuildFailed(benchmark.BenchmarkCase, buildResult);
+                }
+            }
+
             return buildResults;
 
             static string GetFormattedDifference(ClockSpan before, ClockSpan after)
@@ -518,12 +557,31 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc
             => logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime(DefaultCultureInfo.Instance)}, executed benchmarks: {executedBenchmarksCount}");
 
         private static BenchmarkRunInfo[] GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, ILogger logger, IResolver resolver)
-            => benchmarkRunInfos.Select(info => new BenchmarkRunInfo(
-                    info.BenchmarksCases.Where(benchmark => benchmark.GetToolchain().IsSupported(benchmark, logger, resolver)).ToArray(),
-                    info.Type,
-                    info.Config))
-                .Where(infos => infos.BenchmarksCases.Any())
-                .ToArray();
+        {
+            var newInfos = new List<BenchmarkRunInfo>(benchmarkRunInfos.Length);
+
+            foreach (var info in benchmarkRunInfos)
+            {
+                var newCases = new List<BenchmarkCase>(info.BenchmarksCases.Length);
+                var eventHandler = info.Config.GetCompositeEventHandler();
+
+                foreach (var benchmark in info.BenchmarksCases)
+                {
+                    if (benchmark.GetToolchain().IsSupported(benchmark, logger, resolver))
+                    {
+                        newCases.Add(benchmark);
+                    }
+                    else
+                    {
+                        eventHandler.HandleUnsupportedBenchmark(benchmark);
+                    }
+                }
+
+                newInfos.Add(new BenchmarkRunInfo(newCases.ToArray(), info.Type, info.Config));
+            }
+
+            return newInfos.ToArray();
+        }
 
         private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos)
         {
@@ -586,6 +644,16 @@ void AddLogger(ILogger logger)
             return new CompositeLogger(loggers.Values.ToImmutableHashSet());
         }
 
+        private static IBenchmarkEventHandler CreateCompositeEventHandler(BenchmarkRunInfo[] benchmarkRunInfos)
+        {
+            var eventHandlers = new HashSet<IBenchmarkEventHandler>();
+
+            foreach (var info in benchmarkRunInfos)
+                eventHandlers.AddRange(info.Config.GetEventHandlers());
+
+            return new CompositeBenchmarkEventHandler(eventHandlers);
+        }
+
         private static void Cleanup(HashSet<string> artifactsToCleanup)
         {
             foreach (string path in artifactsToCleanup)

src/BenchmarkDotNet/Running/CompositeBenchmarkEventHandler.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Toolchains.Results;
+using BenchmarkDotNet.Validators;
+
+namespace BenchmarkDotNet.Running
+{
+    public class CompositeBenchmarkEventHandler : IBenchmarkEventHandler
+    {
+        private readonly IReadOnlyCollection<IBenchmarkEventHandler> eventHandlers;
+
+        public CompositeBenchmarkEventHandler(IReadOnlyCollection<IBenchmarkEventHandler> eventHandlers)
+        {
+            this.eventHandlers = eventHandlers;
+        }
+
+        public void HandleStartValidationStage()
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleStartValidationStage();
+        }
+
+        public void HandleUnsupportedBenchmark(BenchmarkCase benchmarkCase)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleUnsupportedBenchmark(benchmarkCase);
+        }
+
+        public void HandleValidationError(ValidationError validationError)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleValidationError(validationError);
+        }
+
+        public void HandleStartBuildStage()
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleStartBuildStage();
+        }
+
+        public void HandleBuildFailed(BenchmarkCase benchmarkCase, BuildResult buildResult)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleBuildFailed(benchmarkCase, buildResult);
+        }
+
+        public void HandleStartRunStage()
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleStartRunStage();
+        }
+
+        public void HandleRunBenchmarksInType(Type type, IReadOnlyList<BenchmarkCase> benchmarks)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleRunBenchmarksInType(type, benchmarks);
+        }
+
+        public void HandleCompletedBenchmarksInType(Type type, Summary summary)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleCompletedBenchmarksInType(type, summary);
+        }
+
+        public void HandleCompletedBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleCompletedBenchmark(benchmarkCase, report);
+        }
+
+        public void HandleRunBenchmark(BenchmarkCase benchmarkCase)
+        {
+            foreach (var eventHandler in eventHandlers)
+                eventHandler.HandleRunBenchmark(benchmarkCase);
+        }
+    }
+}