From 202a8d19bfe680a0529b8619933f310b6e0b211e Mon Sep 17 00:00:00 2001 From: alhardy Date: Sat, 31 Mar 2018 22:33:47 +1100 Subject: [PATCH] AppMetrics\AppMetrics#243 #5 Adding reporting hosted service and configuration extensions --- .../Host.cs | 28 +-- .../Host.cs | 53 +++--- ... MetricsConfigureHostBuilderExtensions.cs} | 40 ++++- .../MetricsReporterBackgroundService.cs | 162 ++++++++++++++++++ ...iceCollectionMetricsReportingExtensions.cs | 42 +++++ 5 files changed, 280 insertions(+), 45 deletions(-) rename src/App.Metrics.Extensions.Hosting/{MetricsHostBuilderExtensions.cs => MetricsConfigureHostBuilderExtensions.cs} (75%) create mode 100644 src/App.Metrics.Extensions.Hosting/Reporting/MetricsReporterBackgroundService.cs create mode 100644 src/App.Metrics.Extensions.Hosting/Reporting/ServiceCollectionMetricsReportingExtensions.cs diff --git a/sandbox/HealthHostingMicrosoftExtensionsSandbox/Host.cs b/sandbox/HealthHostingMicrosoftExtensionsSandbox/Host.cs index 5cd8cac..767ba03 100644 --- a/sandbox/HealthHostingMicrosoftExtensionsSandbox/Host.cs +++ b/sandbox/HealthHostingMicrosoftExtensionsSandbox/Host.cs @@ -29,19 +29,21 @@ public static async Task Main(string[] args) .CreateLogger(); var host = new HostBuilder().ConfigureAppConfiguration( - (hostContext, config) => - { - config.SetBasePath(Directory.GetCurrentDirectory()); - config.AddEnvironmentVariables(); - config.AddJsonFile("appsettings.json", optional: true); - config.AddCommandLine(args); - }).ConfigureHealthWithDefaults( - (context, builder) => - { - builder.OutputHealth.AsPlainText() - .OutputHealth.AsJson() - .HealthChecks.AddCheck("inline-check", () => new ValueTask(HealthCheckResult.Healthy())); - }).Build(); + (hostContext, config) => + { + config.SetBasePath(Directory.GetCurrentDirectory()); + config.AddEnvironmentVariables(); + config.AddJsonFile("appsettings.json", optional: true); + config.AddCommandLine(args); + }) + .ConfigureHealthWithDefaults( + (context, builder) => + { + builder.OutputHealth.AsPlainText() + .OutputHealth.AsJson() + .HealthChecks.AddCheck("inline-check", () => new ValueTask(HealthCheckResult.Healthy())); + }) + .Build(); var health = host.Services.GetRequiredService(); diff --git a/sandbox/MetricsHostingMicrosoftExtensionsSandbox/Host.cs b/sandbox/MetricsHostingMicrosoftExtensionsSandbox/Host.cs index 4ccd401..6cda7c1 100644 --- a/sandbox/MetricsHostingMicrosoftExtensionsSandbox/Host.cs +++ b/sandbox/MetricsHostingMicrosoftExtensionsSandbox/Host.cs @@ -7,9 +7,8 @@ using System.Threading; using System.Threading.Tasks; using App.Metrics; -using App.Metrics.Extensions.Configuration; using App.Metrics.Extensions.Hosting; -using App.Metrics.Reporting; +using App.Metrics.Scheduling; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -23,28 +22,29 @@ public class Host { public static async Task Main(string[] args) { - Log.Logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.LiterateConsole(LogEventLevel.Information).WriteTo. - Seq("http://localhost:5341", LogEventLevel.Verbose).CreateLogger(); + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.LiterateConsole(LogEventLevel.Information) + .WriteTo.Seq("http://localhost:5341", LogEventLevel.Verbose) + .CreateLogger(); - var host = new HostBuilder().ConfigureAppConfiguration( - (hostContext, config) => - { - config.SetBasePath(Directory.GetCurrentDirectory()); - config.AddEnvironmentVariables(); - config.AddJsonFile("appsettings.json", optional: true); - config.AddCommandLine(args); - }).ConfigureMetricsWithDefaults( - (context, builder) => - { - builder.Configuration.ReadFrom(context.Configuration); - builder.OutputEnvInfo.AsPlainText(); - builder.OutputMetrics.AsPlainText(); - builder.OutputMetrics.AsJson(); - builder.Report.Using(TimeSpan.FromSeconds(5)); - }).Build(); + var host = new HostBuilder() + .ConfigureAppConfiguration( + (hostContext, config) => + { + config.SetBasePath(Directory.GetCurrentDirectory()); + config.AddEnvironmentVariables(); + config.AddJsonFile("appsettings.json", optional: true); + config.AddCommandLine(args); + }) + .ConfigureMetrics( + (context, builder) => + { + builder.Report.Using(TimeSpan.FromSeconds(5)); + }) + .Build(); var metrics = host.Services.GetRequiredService(); - var reporter = host.Services.GetRequiredService(); var cancellationTokenSource = new CancellationTokenSource(); @@ -56,16 +56,17 @@ public static async Task Main(string[] args) host.PressAnyKeyToContinue(); - await host.RunUntilEscAsync( - TimeSpan.FromSeconds(5), - cancellationTokenSource, - async () => + var recordMetricsTask = new AppMetricsTaskScheduler( + TimeSpan.FromSeconds(2), + () => { Clear(); host.RecordMetrics(metrics); - await Task.WhenAll(reporter.RunAllAsync(cancellationTokenSource.Token)); + return Task.CompletedTask; }); + recordMetricsTask.Start(); + await host.RunAsync(token: cancellationTokenSource.Token); } } diff --git a/src/App.Metrics.Extensions.Hosting/MetricsHostBuilderExtensions.cs b/src/App.Metrics.Extensions.Hosting/MetricsConfigureHostBuilderExtensions.cs similarity index 75% rename from src/App.Metrics.Extensions.Hosting/MetricsHostBuilderExtensions.cs rename to src/App.Metrics.Extensions.Hosting/MetricsConfigureHostBuilderExtensions.cs index d90d0fe..00b796c 100644 --- a/src/App.Metrics.Extensions.Hosting/MetricsHostBuilderExtensions.cs +++ b/src/App.Metrics.Extensions.Hosting/MetricsConfigureHostBuilderExtensions.cs @@ -1,15 +1,16 @@ -// +// // Copyright (c) Allan Hardy. All rights reserved. // using System; +using System.Linq; using App.Metrics.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; namespace App.Metrics.Extensions.Hosting { - public static class MetricsHostBuilderExtensions + public static class MetricsConfigureHostBuilderExtensions { private static bool _metricsBuilt; @@ -26,14 +27,24 @@ public static IHostBuilder ConfigureMetricsWithDefaults( (context, services) => { var metricsBuilder = AppMetrics.CreateDefaultBuilder(); + configureMetrics(context, metricsBuilder); + metricsBuilder.Configuration.ReadFrom(context.Configuration); + + if (metricsBuilder.CanReport()) + { + services.AddMetricsReportingHostedService(); + } + services.AddMetrics(metricsBuilder); _metricsBuilt = true; }); } - public static IHostBuilder ConfigureMetricsWithDefaults(this IHostBuilder hostBuilder, Action configureMetrics) + public static IHostBuilder ConfigureMetricsWithDefaults( + this IHostBuilder hostBuilder, + Action configureMetrics) { if (_metricsBuilt) { @@ -49,7 +60,9 @@ public static IHostBuilder ConfigureMetricsWithDefaults(this IHostBuilder hostBu return hostBuilder; } - public static IHostBuilder ConfigureMetrics(this IHostBuilder hostBuilder, IMetricsRoot metrics) + public static IHostBuilder ConfigureMetrics( + this IHostBuilder hostBuilder, + IMetricsRoot metrics) { if (_metricsBuilt) { @@ -59,7 +72,13 @@ public static IHostBuilder ConfigureMetrics(this IHostBuilder hostBuilder, IMetr return hostBuilder.ConfigureServices( (context, services) => { + if (metrics.Options.ReportingEnabled && metrics.Reporters != null && metrics.Reporters.Any()) + { + services.AddMetricsReportingHostedService(); + } + services.AddMetrics(metrics); + _metricsBuilt = true; }); } @@ -81,12 +100,20 @@ public static IHostBuilder ConfigureMetrics( { configureMetrics(context, builder); builder.Configuration.ReadFrom(context.Configuration); + + if (builder.CanReport()) + { + services.AddMetricsReportingHostedService(); + } + _metricsBuilt = true; }); }); } - public static IHostBuilder ConfigureMetrics(this IHostBuilder hostBuilder, Action configureMetrics) + public static IHostBuilder ConfigureMetrics( + this IHostBuilder hostBuilder, + Action configureMetrics) { if (_metricsBuilt) { @@ -115,7 +142,8 @@ public static IHostBuilder ConfigureMetrics(this IHostBuilder hostBuilder) if (!_metricsBuilt) { var builder = AppMetrics.CreateDefaultBuilder() - .Configuration.ReadFrom(context.Configuration); + .Configuration.ReadFrom(context.Configuration); + services.AddMetrics(builder); _metricsBuilt = true; } diff --git a/src/App.Metrics.Extensions.Hosting/Reporting/MetricsReporterBackgroundService.cs b/src/App.Metrics.Extensions.Hosting/Reporting/MetricsReporterBackgroundService.cs new file mode 100644 index 0000000..4061e85 --- /dev/null +++ b/src/App.Metrics.Extensions.Hosting/Reporting/MetricsReporterBackgroundService.cs @@ -0,0 +1,162 @@ +// +// Copyright (c) Allan Hardy. All rights reserved. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using App.Metrics.Counter; +using App.Metrics.Logging; +using App.Metrics.Reporting; +using Microsoft.Extensions.Hosting; + +namespace App.Metrics.Extensions.Hosting.Reporting +{ + public class MetricsReporterBackgroundService : BackgroundService + { + private static readonly ILog Logger = LogProvider.For(); + private static readonly TimeSpan WaitBetweenReportRunChecks = TimeSpan.FromMilliseconds(500); + private readonly IMetrics _metrics; + private readonly CounterOptions _successCounter; + private readonly CounterOptions _failedCounter; + private readonly MetricsOptions _options; + private readonly List _scheduledReporters = new List(); + + public MetricsReporterBackgroundService( + IMetrics metrics, + MetricsOptions options, + IEnumerable reporters) + { + _metrics = metrics; + _options = options; + + var referenceTime = DateTime.UtcNow; + + _successCounter = new CounterOptions + { + Context = AppMetricsConstants.InternalMetricsContext, + MeasurementUnit = Unit.Items, + ResetOnReporting = true, + Name = "report_success" + }; + + _failedCounter = new CounterOptions + { + Context = AppMetricsConstants.InternalMetricsContext, + MeasurementUnit = Unit.Items, + ResetOnReporting = true, + Name = "report_failed" + }; + + foreach (var reporter in reporters) + { + _scheduledReporters.Add( + new SchedulerTaskWrapper + { + Interval = reporter.FlushInterval, + Reporter = reporter, + NextRunTime = referenceTime + }); + } + } + + public event EventHandler UnobservedTaskException; + + protected override async Task ExecuteAsync(CancellationToken cancellationToken) + { + if (!_scheduledReporters.Any()) + { + await Task.CompletedTask; + } + + while (!cancellationToken.IsCancellationRequested + && _options.Enabled + && _options.ReportingEnabled) + { + await ExecuteOnceAsync(cancellationToken); + + Logger.Trace($"Delaying for {WaitBetweenReportRunChecks}"); + await Task.Delay(WaitBetweenReportRunChecks, cancellationToken); + } + } + + private async Task ExecuteOnceAsync(CancellationToken cancellationToken) + { + var taskFactory = new TaskFactory(TaskScheduler.Current); + var referenceTime = DateTime.UtcNow; + + foreach (var flushTask in _scheduledReporters) + { + if (!flushTask.ShouldRun(referenceTime)) + { + Logger.Trace($"Skipping {flushTask.Reporter.GetType().FullName}, next run in {flushTask.NextRunTime.Subtract(referenceTime).Milliseconds} ms"); + continue; + } + + flushTask.Increment(); + + await taskFactory.StartNew( + async () => + { + try + { + Logger.Trace($"Executing reporter {flushTask.Reporter.GetType().FullName} FlushAsync"); + + var result = await flushTask.Reporter.FlushAsync( + _metrics.Snapshot.Get(flushTask.Reporter.Filter), + cancellationToken); + + if (result) + { + _metrics.Measure.Counter.Increment(_successCounter, flushTask.Reporter.GetType().FullName); + Logger.Trace($"Reporter {flushTask.Reporter.GetType().FullName} FlushAsync executed successfully"); + } + else + { + _metrics.Measure.Counter.Increment(_failedCounter, flushTask.Reporter.GetType().FullName); + Logger.Warn($"Reporter {flushTask.Reporter.GetType().FullName} FlushAsync failed"); + } + } + catch (Exception ex) + { + _metrics.Measure.Counter.Increment(_failedCounter, flushTask.Reporter.GetType().FullName); + + var args = new UnobservedTaskExceptionEventArgs( + ex as AggregateException ?? new AggregateException(ex)); + + Logger.Error($"Reporter {flushTask.Reporter.GetType().FullName} FlushAsync failed", ex); + + UnobservedTaskException?.Invoke(this, args); + + if (!args.Observed) + { + throw; + } + } + }, + cancellationToken); + } + } + + private class SchedulerTaskWrapper + { + public TimeSpan Interval { get; set; } + + public DateTime LastRunTime { get; set; } + + public DateTime NextRunTime { get; set; } + + public IReportMetrics Reporter { get; set; } + + public void Increment() + { + LastRunTime = NextRunTime; + NextRunTime = DateTime.UtcNow.Add(Interval); + } + + public bool ShouldRun(DateTime currentTime) { return NextRunTime < currentTime && LastRunTime != NextRunTime; } + } + } +} \ No newline at end of file diff --git a/src/App.Metrics.Extensions.Hosting/Reporting/ServiceCollectionMetricsReportingExtensions.cs b/src/App.Metrics.Extensions.Hosting/Reporting/ServiceCollectionMetricsReportingExtensions.cs new file mode 100644 index 0000000..0ccb7d5 --- /dev/null +++ b/src/App.Metrics.Extensions.Hosting/Reporting/ServiceCollectionMetricsReportingExtensions.cs @@ -0,0 +1,42 @@ +// +// Copyright (c) Allan Hardy. All rights reserved. +// + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using App.Metrics; +using App.Metrics.Extensions.Hosting.Reporting; +using App.Metrics.Reporting; +using Microsoft.Extensions.Hosting; + +// ReSharper disable CheckNamespace +namespace Microsoft.Extensions.DependencyInjection + // ReSharper restore CheckNamespace +{ + public static class ServiceCollectionMetricsReportingExtensions + { + public static IServiceCollection AddMetricsReportingHostedService( + this IServiceCollection services, + EventHandler unobservedTaskExceptionHandler = null) + { + services.AddSingleton(serviceProvider => + { + var options = serviceProvider.GetRequiredService(); + var metrics = serviceProvider.GetRequiredService(); + var reporters = serviceProvider.GetService>(); + + var instance = new MetricsReporterBackgroundService(metrics, options, reporters); + + if (unobservedTaskExceptionHandler != null) + { + instance.UnobservedTaskException += unobservedTaskExceptionHandler; + } + + return instance; + }); + + return services; + } + } +}