From 08531c454b4cca656dedec05afb04b8d010f5b08 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 8 Apr 2021 11:01:08 -0700 Subject: [PATCH 01/45] Adding logging --- .../src/HttpLoggingBuilderExtensions.cs | 40 ++ .../HttpLogging2/src/HttpLoggingExtensions.cs | 59 +++ .../HttpLogging2/src/HttpLoggingMiddleware.cs | 174 +++++++ .../HttpLogging2/src/HttpLoggingOptions.cs | 29 ++ .../src/HttpLoggingServicesExtensions.cs | 35 ++ .../Microsoft.AspNetCore.HttpLogging.csproj | 22 + .../HttpLogging2/src/PublicAPI.Shipped.txt | 1 + .../HttpLogging2/src/PublicAPI.Unshipped.txt | 1 + .../HttpLogging2/test/HstsMiddlewareTests.cs | 407 +++++++++++++++ .../HttpLogging2/test/HttpsPolicyTests.cs | 86 ++++ .../test/HttpsRedirectionMiddlewareTests.cs | 462 ++++++++++++++++++ ...rosoft.AspNetCore.HttpsPolicy.Tests.csproj | 12 + 12 files changed, 1328 insertions(+) create mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs create mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs create mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs create mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs create mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs create mode 100644 src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj create mode 100644 src/Middleware/HttpLogging2/src/PublicAPI.Shipped.txt create mode 100644 src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt create mode 100644 src/Middleware/HttpLogging2/test/HstsMiddlewareTests.cs create mode 100644 src/Middleware/HttpLogging2/test/HttpsPolicyTests.cs create mode 100644 src/Middleware/HttpLogging2/test/HttpsRedirectionMiddlewareTests.cs create mode 100644 src/Middleware/HttpLogging2/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs new file mode 100644 index 000000000000..c6f804763fff --- /dev/null +++ b/src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.HttpsPolicy; + +namespace Microsoft.AspNetCore.Builder +{ + /// + /// Extension methods for the HttpsRedirection middleware. + /// + public static class HttpsPolicyBuilderExtensions + { + /// + /// Adds middleware for redirecting HTTP Requests to HTTPS. + /// + /// The instance this method extends. + /// The for HttpsRedirection. + public static IApplicationBuilder UseHttpsRedirection(this IApplicationBuilder app) + { + if (app == null) + { + throw new ArgumentNullException(nameof(app)); + } + + var serverAddressFeature = app.ServerFeatures.Get(); + if (serverAddressFeature != null) + { + app.UseMiddleware(serverAddressFeature); + } + else + { + app.UseMiddleware(); + } + return app; + } + } +} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs new file mode 100644 index 000000000000..f3b00416f566 --- /dev/null +++ b/src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.HttpsPolicy +{ + internal static class HttpsLoggingExtensions + { + private static readonly Action _redirectingToHttps; + private static readonly Action _portLoadedFromConfig; + private static readonly Action _failedToDeterminePort; + private static readonly Action _portFromServer; + + static HttpsLoggingExtensions() + { + _redirectingToHttps = LoggerMessage.Define( + LogLevel.Debug, + new EventId(1, "RedirectingToHttps"), + "Redirecting to '{redirect}'."); + + _portLoadedFromConfig = LoggerMessage.Define( + LogLevel.Debug, + new EventId(2, "PortLoadedFromConfig"), + "Https port '{port}' loaded from configuration."); + + _failedToDeterminePort = LoggerMessage.Define( + LogLevel.Warning, + new EventId(3, "FailedToDeterminePort"), + "Failed to determine the https port for redirect."); + + _portFromServer = LoggerMessage.Define( + LogLevel.Debug, + new EventId(5, "PortFromServer"), + "Https port '{httpsPort}' discovered from server endpoints."); + } + + public static void RedirectingToHttps(this ILogger logger, string redirect) + { + _redirectingToHttps(logger, redirect, null); + } + + public static void PortLoadedFromConfig(this ILogger logger, int port) + { + _portLoadedFromConfig(logger, port, null); + } + + public static void FailedToDeterminePort(this ILogger logger) + { + _failedToDeterminePort(logger, null); + } + + public static void PortFromServer(this ILogger logger, int port) + { + _portFromServer(logger, port, null); + } + } +} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs new file mode 100644 index 000000000000..215507912f3e --- /dev/null +++ b/src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs @@ -0,0 +1,174 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Net.Http.Headers; + +namespace Microsoft.AspNetCore.HttpsPolicy +{ + /// + /// Middleware that redirects non-HTTPS requests to an HTTPS URL. + /// + public class HttpsRedirectionMiddleware + { + private const int PortNotFound = -1; + + private readonly RequestDelegate _next; + private readonly Lazy _httpsPort; + private readonly int _statusCode; + + private readonly IServerAddressesFeature? _serverAddressesFeature; + private readonly IConfiguration _config; + private readonly ILogger _logger; + + /// + /// Initializes . + /// + /// + /// + /// + /// + public HttpsRedirectionMiddleware(RequestDelegate next, IOptions options, IConfiguration config, ILoggerFactory loggerFactory) + + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + var httpsRedirectionOptions = options.Value; + if (httpsRedirectionOptions.HttpsPort.HasValue) + { + _httpsPort = new Lazy(httpsRedirectionOptions.HttpsPort.Value); + } + else + { + _httpsPort = new Lazy(TryGetHttpsPort); + } + _statusCode = httpsRedirectionOptions.RedirectStatusCode; + _logger = loggerFactory.CreateLogger(); + } + + /// + /// Initializes . + /// + /// + /// + /// + /// + /// + public HttpsRedirectionMiddleware(RequestDelegate next, IOptions options, IConfiguration config, ILoggerFactory loggerFactory, + IServerAddressesFeature serverAddressesFeature) + : this(next, options, config, loggerFactory) + { + _serverAddressesFeature = serverAddressesFeature ?? throw new ArgumentNullException(nameof(serverAddressesFeature)); + } + + /// + /// Invokes the HttpsRedirectionMiddleware. + /// + /// + /// + public Task Invoke(HttpContext context) + { + if (context.Request.IsHttps) + { + return _next(context); + } + + var port = _httpsPort.Value; + if (port == PortNotFound) + { + return _next(context); + } + + var host = context.Request.Host; + if (port != 443) + { + host = new HostString(host.Host, port); + } + else + { + host = new HostString(host.Host); + } + + var request = context.Request; + var redirectUrl = UriHelper.BuildAbsolute( + "https", + host, + request.PathBase, + request.Path, + request.QueryString); + + context.Response.StatusCode = _statusCode; + context.Response.Headers[HeaderNames.Location] = redirectUrl; + + _logger.RedirectingToHttps(redirectUrl); + + return Task.CompletedTask; + } + + // Returns PortNotFound (-1) if we were unable to determine the port. + private int TryGetHttpsPort() + { + // The IServerAddressesFeature will not be ready until the middleware is Invoked, + // Order for finding the HTTPS port: + // 1. Set in the HttpsRedirectionOptions + // 2. HTTPS_PORT environment variable + // 3. IServerAddressesFeature + // 4. Fail if not sets + + var nullablePort = _config.GetValue("HTTPS_PORT") ?? _config.GetValue("ANCM_HTTPS_PORT"); + if (nullablePort.HasValue) + { + var port = nullablePort.Value; + _logger.PortLoadedFromConfig(port); + return port; + } + + if (_serverAddressesFeature == null) + { + _logger.FailedToDeterminePort(); + return PortNotFound; + } + + foreach (var address in _serverAddressesFeature.Addresses) + { + var bindingAddress = BindingAddress.Parse(address); + if (bindingAddress.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + // If we find multiple different https ports specified, throw + if (nullablePort.HasValue && nullablePort != bindingAddress.Port) + { + throw new InvalidOperationException( + "Cannot determine the https port from IServerAddressesFeature, multiple values were found. " + + "Set the desired port explicitly on HttpsRedirectionOptions.HttpsPort."); + } + else + { + nullablePort = bindingAddress.Port; + } + } + } + + if (nullablePort.HasValue) + { + var port = nullablePort.Value; + _logger.PortFromServer(port); + return port; + } + + _logger.FailedToDeterminePort(); + return PortNotFound; + } + } +} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs new file mode 100644 index 000000000000..0e1c625bd7dc --- /dev/null +++ b/src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.AspNetCore.Http; + +namespace Microsoft.AspNetCore.HttpsPolicy +{ + /// + /// Options for the HTTPS Redirection Middleware. + /// + public class HttpsRedirectionOptions + { + /// + /// The status code used for the redirect response. The default is 307. + /// + public int RedirectStatusCode { get; set; } = StatusCodes.Status307TemporaryRedirect; + + /// + /// The HTTPS port to be added to the redirected URL. + /// + /// + /// If the HttpsPort is not set, we will try to get the HttpsPort from the following: + /// 1. HTTPS_PORT environment variable + /// 2. IServerAddressesFeature + /// If that fails then the middleware will log a warning and turn off. + /// + public int? HttpsPort { get; set; } + } +} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs new file mode 100644 index 000000000000..24c9e114e8b9 --- /dev/null +++ b/src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.HttpsPolicy; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.AspNetCore.Builder +{ + /// + /// Extension methods for the HttpsRedirection middleware. + /// + public static class HttpsRedirectionServicesExtensions + { + /// + /// Adds HTTPS redirection services. + /// + /// The for adding services. + /// A delegate to configure the . + /// + public static IServiceCollection AddHttpsRedirection(this IServiceCollection services, Action configureOptions) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + if (configureOptions == null) + { + throw new ArgumentNullException(nameof(configureOptions)); + } + services.Configure(configureOptions); + return services; + } + } +} diff --git a/src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj new file mode 100644 index 000000000000..c14e35c11fda --- /dev/null +++ b/src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -0,0 +1,22 @@ + + + + + ASP.NET Core basic middleware for supporting HTTPS Redirection and HTTP Strict-Transport-Security. + + $(DefaultNetCoreTargetFramework) + true + true + aspnetcore;https;hsts + false + enable + + + + + + + + + + diff --git a/src/Middleware/HttpLogging2/src/PublicAPI.Shipped.txt b/src/Middleware/HttpLogging2/src/PublicAPI.Shipped.txt new file mode 100644 index 000000000000..7dc5c58110bf --- /dev/null +++ b/src/Middleware/HttpLogging2/src/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt new file mode 100644 index 000000000000..7dc5c58110bf --- /dev/null +++ b/src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/Middleware/HttpLogging2/test/HstsMiddlewareTests.cs b/src/Middleware/HttpLogging2/test/HstsMiddlewareTests.cs new file mode 100644 index 000000000000..232f2e474803 --- /dev/null +++ b/src/Middleware/HttpLogging2/test/HstsMiddlewareTests.cs @@ -0,0 +1,407 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using Microsoft.Net.Http.Headers; +using Xunit; + +namespace Microsoft.AspNetCore.HttpsPolicy.Tests +{ + public class HstsMiddlewareTests + { + [Fact] + public async Task SetOptionsWithDefault_SetsMaxAgeToCorrectValue() + { + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri("https://example.com:5050"); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("max-age=2592000", response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); + } + + [Theory] + [InlineData(0, false, false, "max-age=0")] + [InlineData(-1, false, false, "max-age=-1")] + [InlineData(0, true, false, "max-age=0; includeSubDomains")] + [InlineData(50000, false, true, "max-age=50000; preload")] + [InlineData(0, true, true, "max-age=0; includeSubDomains; preload")] + [InlineData(50000, true, true, "max-age=50000; includeSubDomains; preload")] + public async Task SetOptionsThroughConfigure_SetsHeaderCorrectly(int maxAge, bool includeSubDomains, bool preload, string expected) + { + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.Configure(options => + { + options.Preload = preload; + options.IncludeSubDomains = includeSubDomains; + options.MaxAge = TimeSpan.FromSeconds(maxAge); + }); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri("https://example.com:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(expected, response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); + } + + [Theory] + [InlineData(0, false, false, "max-age=0")] + [InlineData(-1, false, false, "max-age=-1")] + [InlineData(0, true, false, "max-age=0; includeSubDomains")] + [InlineData(50000, false, true, "max-age=50000; preload")] + [InlineData(0, true, true, "max-age=0; includeSubDomains; preload")] + [InlineData(50000, true, true, "max-age=50000; includeSubDomains; preload")] + public async Task SetOptionsThroughHelper_SetsHeaderCorrectly(int maxAge, bool includeSubDomains, bool preload, string expected) + { + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddHsts(options => + { + options.Preload = preload; + options.IncludeSubDomains = includeSubDomains; + options.MaxAge = TimeSpan.FromSeconds(maxAge); + }); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri("https://example.com:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(expected, response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); + } + + [Theory] + [InlineData("localhost")] + [InlineData("Localhost")] + [InlineData("LOCALHOST")] + [InlineData("127.0.0.1")] + [InlineData("[::1]")] + public async Task DefaultExcludesCommonLocalhostDomains_DoesNotSetHstsHeader(string hostUrl) + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri($"https://{hostUrl}:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Headers); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal($"The host '{hostUrl}' is excluded. Skipping HSTS header.", message.State.ToString(), ignoreCase: true); + } + + [Theory] + [InlineData("localhost")] + [InlineData("127.0.0.1")] + [InlineData("[::1]")] + public async Task AllowLocalhostDomainsIfListIsReset_SetHstsHeader(string hostUrl) + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + + services.AddHsts(options => + { + options.ExcludedHosts.Clear(); + }); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri($"https://{hostUrl}:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Single(response.Headers); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Trace, message.LogLevel); + Assert.Equal("Adding HSTS header to response.", message.State.ToString()); + } + + [Theory] + [InlineData("example.com")] + [InlineData("Example.com")] + [InlineData("EXAMPLE.COM")] + public async Task AddExcludedDomains_DoesNotAddHstsHeader(string hostUrl) + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + + services.AddHsts(options => + { + options.ExcludedHosts.Add(hostUrl); + }); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri($"https://{hostUrl}:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Headers); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal($"The host '{hostUrl}' is excluded. Skipping HSTS header.", message.State.ToString(), ignoreCase: true); + } + + [Fact] + public async Task WhenRequestIsInsecure_DoesNotAddHstsHeader() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri("http://example.com:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Headers); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal("The request is insecure. Skipping HSTS header.", message.State.ToString()); + } + + [Fact] + public async Task WhenRequestIsSecure_AddsHstsHeader() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + var server = host.GetTestServer(); + var client = server.CreateClient(); + client.BaseAddress = new Uri("https://example.com:5050"); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(response.Headers, x => x.Key == HeaderNames.StrictTransportSecurity); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Trace, message.LogLevel); + Assert.Equal("Adding HSTS header to response.", message.State.ToString()); + } + } +} diff --git a/src/Middleware/HttpLogging2/test/HttpsPolicyTests.cs b/src/Middleware/HttpLogging2/test/HttpsPolicyTests.cs new file mode 100644 index 000000000000..d246c6255f01 --- /dev/null +++ b/src/Middleware/HttpLogging2/test/HttpsPolicyTests.cs @@ -0,0 +1,86 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Net.Http.Headers; +using Xunit; + +namespace Microsoft.AspNetCore.HttpsPolicy.Tests +{ + public class HttpsPolicyTests + { + [Theory] + [InlineData(302, 443, 2592000, false, false, "max-age=2592000", "https://localhost/")] + [InlineData(301, 5050, 2592000, false, false, "max-age=2592000", "https://localhost:5050/")] + [InlineData(301, 443, 2592000, false, false, "max-age=2592000", "https://localhost/")] + [InlineData(301, 443, 2592000, true, false, "max-age=2592000; includeSubDomains", "https://localhost/")] + [InlineData(301, 443, 2592000, false, true, "max-age=2592000; preload", "https://localhost/")] + [InlineData(301, 443, 2592000, true, true, "max-age=2592000; includeSubDomains; preload", "https://localhost/")] + [InlineData(302, 5050, 2592000, true, true, "max-age=2592000; includeSubDomains; preload", "https://localhost:5050/")] + public async Task SetsBothHstsAndHttpsRedirection_RedirectOnFirstRequest_HstsOnSecondRequest(int statusCode, int? tlsPort, int maxAge, bool includeSubDomains, bool preload, string expectedHstsHeader, string expectedUrl) + { + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.Configure(options => + { + options.RedirectStatusCode = statusCode; + options.HttpsPort = tlsPort; + }); + services.Configure(options => + { + options.IncludeSubDomains = includeSubDomains; + options.MaxAge = TimeSpan.FromSeconds(maxAge); + options.Preload = preload; + options.ExcludedHosts.Clear(); // allowing localhost for testing + }); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.UseHsts(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + server.Features.Set(new ServerAddressesFeature()); + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(statusCode, (int)response.StatusCode); + Assert.Equal(expectedUrl, response.Headers.Location.ToString()); + + client = server.CreateClient(); + client.BaseAddress = new Uri(response.Headers.Location.ToString()); + request = new HttpRequestMessage(HttpMethod.Get, expectedUrl); + response = await client.SendAsync(request); + + Assert.Equal(expectedHstsHeader, response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); + } + } +} diff --git a/src/Middleware/HttpLogging2/test/HttpsRedirectionMiddlewareTests.cs b/src/Middleware/HttpLogging2/test/HttpsRedirectionMiddlewareTests.cs new file mode 100644 index 000000000000..d4bd8fc7df2e --- /dev/null +++ b/src/Middleware/HttpLogging2/test/HttpsRedirectionMiddlewareTests.cs @@ -0,0 +1,462 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using Xunit; + +namespace Microsoft.AspNetCore.HttpsPolicy.Tests +{ + public class HttpsRedirectionMiddlewareTests + { + [Fact] + public async Task SetOptions_NotEnabledByDefault() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Warning, message.LogLevel); + Assert.Equal("Failed to determine the https port for redirect.", message.State.ToString()); + } + + [Theory] + [InlineData(302, 5001, "https://localhost:5001/")] + [InlineData(307, 1, "https://localhost:1/")] + [InlineData(308, 3449, "https://localhost:3449/")] + [InlineData(301, 5050, "https://localhost:5050/")] + [InlineData(301, 443, "https://localhost/")] + public async Task SetOptions_SetStatusCodeHttpsPort(int statusCode, int? httpsPort, string expected) + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + services.Configure(options => + { + options.RedirectStatusCode = statusCode; + options.HttpsPort = httpsPort; + }); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(statusCode, (int)response.StatusCode); + Assert.Equal(expected, response.Headers.Location.ToString()); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal($"Redirecting to '{expected}'.", message.State.ToString()); + } + + [Theory] + [InlineData(302, 5001, "https://localhost:5001/")] + [InlineData(307, 1, "https://localhost:1/")] + [InlineData(308, 3449, "https://localhost:3449/")] + [InlineData(301, 5050, "https://localhost:5050/")] + [InlineData(301, 443, "https://localhost/")] + public async Task SetOptionsThroughHelperMethod_SetStatusCodeAndHttpsPort(int statusCode, int? httpsPort, string expectedUrl) + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + services.AddHttpsRedirection(options => + { + options.RedirectStatusCode = statusCode; + options.HttpsPort = httpsPort; + }); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(statusCode, (int)response.StatusCode); + Assert.Equal(expectedUrl, response.Headers.Location.ToString()); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.Single(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal($"Redirecting to '{expectedUrl}'.", message.State.ToString()); + } + + [Theory] + [InlineData(null, null, "https://localhost:4444/", "https://localhost:4444/")] + [InlineData(null, null, "https://localhost:443/", "https://localhost/")] + [InlineData(null, null, "https://localhost/", "https://localhost/")] + [InlineData(null, "5000", "https://localhost:4444/", "https://localhost:5000/")] + [InlineData(null, "443", "https://localhost:4444/", "https://localhost/")] + [InlineData(443, "5000", "https://localhost:4444/", "https://localhost/")] + [InlineData(4000, "5000", "https://localhost:4444/", "https://localhost:4000/")] + [InlineData(5000, null, "https://localhost:4444/", "https://localhost:5000/")] + public async Task SetHttpsPortEnvironmentVariableAndServerFeature_ReturnsCorrectStatusCodeOnResponse( + int? optionsHttpsPort, string configHttpsPort, string serverAddressFeatureUrl, string expectedUrl) + { + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddHttpsRedirection(options => + { + options.HttpsPort = optionsHttpsPort; + }); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + + webHostBuilder.UseSetting("HTTPS_PORT", configHttpsPort); + }).Build(); + + var server = host.GetTestServer(); + server.Features.Set(new ServerAddressesFeature()); + if (serverAddressFeatureUrl != null) + { + server.Features.Get().Addresses.Add(serverAddressFeatureUrl); + } + + await host.StartAsync(); + + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal(expectedUrl, response.Headers.Location.ToString()); + } + + [Fact] + public async Task SetServerAddressesFeature_SingleHttpsAddress_Success() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + var server = host.GetTestServer(); + server.Features.Set(new ServerAddressesFeature()); + + server.Features.Get().Addresses.Add("https://localhost:5050"); + await host.StartAsync(); + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal("https://localhost:5050/", response.Headers.Location.ToString()); + + var logMessages = sink.Writes.ToList(); + + Assert.Equal(2, logMessages.Count); + var message = logMessages.First(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal("Https port '5050' discovered from server endpoints.", message.State.ToString()); + + message = logMessages.Skip(1).First(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal("Redirecting to 'https://localhost:5050/'.", message.State.ToString()); + } + + [Fact] + public async Task SetServerAddressesFeature_MultipleHttpsAddresses_Throws() + { + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + var server = host.GetTestServer(); + server.Features.Set(new ServerAddressesFeature()); + + server.Features.Get().Addresses.Add("https://localhost:5050"); + server.Features.Get().Addresses.Add("https://localhost:5051"); + + await host.StartAsync(); + + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var ex = await Assert.ThrowsAsync(() => client.SendAsync(request)); + Assert.Equal("Cannot determine the https port from IServerAddressesFeature, multiple values were found. " + + "Set the desired port explicitly on HttpsRedirectionOptions.HttpsPort.", ex.Message); + } + + [Fact] + public async Task SetServerAddressesFeature_MultipleHttpsAddressesWithSamePort_Success() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + var server = host.GetTestServer(); + server.Features.Set(new ServerAddressesFeature()); + server.Features.Get().Addresses.Add("https://localhost:5050"); + server.Features.Get().Addresses.Add("https://example.com:5050"); + + await host.StartAsync(); + + var client = server.CreateClient(); + + var request = new HttpRequestMessage(HttpMethod.Get, ""); + + var response = await client.SendAsync(request); + + Assert.Equal("https://localhost:5050/", response.Headers.Location.ToString()); + + var logMessages = sink.Writes.ToList(); + + Assert.Equal(2, logMessages.Count); + var message = logMessages.First(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal("Https port '5050' discovered from server endpoints.", message.State.ToString()); + + message = logMessages.Skip(1).First(); + Assert.Equal(LogLevel.Debug, message.LogLevel); + Assert.Equal("Redirecting to 'https://localhost:5050/'.", message.State.ToString()); + } + + [Fact] + public async Task NoServerAddressFeature_DoesNotThrow_DoesNotRedirect() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + await host.StartAsync(); + + var server = host.GetTestServer(); + var client = server.CreateClient(); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + var response = await client.SendAsync(request); + Assert.Equal(200, (int)response.StatusCode); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.First(); + Assert.Equal(LogLevel.Warning, message.LogLevel); + Assert.Equal("Failed to determine the https port for redirect.", message.State.ToString()); + } + + [Fact] + public async Task SetNullAddressFeature_DoesNotThrow() + { + var sink = new TestSink( + TestSink.EnableWithTypeName, + TestSink.EnableWithTypeName); + var loggerFactory = new TestLoggerFactory(sink, enabled: true); + using var host = new HostBuilder() + .ConfigureWebHost(webHostBuilder => + { + webHostBuilder + .UseTestServer() + .ConfigureServices(services => + { + services.AddSingleton(loggerFactory); + }) + .Configure(app => + { + app.UseHttpsRedirection(); + app.Run(context => + { + return context.Response.WriteAsync("Hello world"); + }); + }); + }).Build(); + + var server = host.GetTestServer(); + server.Features.Set(null); + + await host.StartAsync(); + + var client = server.CreateClient(); + var request = new HttpRequestMessage(HttpMethod.Get, ""); + var response = await client.SendAsync(request); + Assert.Equal(200, (int)response.StatusCode); + + var logMessages = sink.Writes.ToList(); + + Assert.Single(logMessages); + var message = logMessages.First(); + Assert.Equal(LogLevel.Warning, message.LogLevel); + Assert.Equal("Failed to determine the https port for redirect.", message.State.ToString()); + } + } +} diff --git a/src/Middleware/HttpLogging2/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj b/src/Middleware/HttpLogging2/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj new file mode 100644 index 000000000000..7691bf5733f1 --- /dev/null +++ b/src/Middleware/HttpLogging2/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj @@ -0,0 +1,12 @@ + + + + $(DefaultNetCoreTargetFramework) + + + + + + + + From b56367f9b613ce43c5d9ab28ad613306be06cf3e Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 9 Apr 2021 09:56:44 -0700 Subject: [PATCH 02/45] Progress --- .../HttpLogging.Sample.csproj | 14 + .../samples/HttpLogging.Sample/Program.cs | 26 ++ .../Properties/launchSettings.json | 28 ++ .../samples/HttpLogging.Sample/Startup.cs | 35 ++ .../appsettings.Development.json | 9 + .../HttpLogging.Sample/appsettings.json | 10 + .../src/HttpLoggingBuilderExtensions.cs | 30 ++ .../HttpLogging/src/HttpLoggingExtensions.cs | 15 + .../HttpLogging/src/HttpLoggingMiddleware.cs | 212 ++++++++++++ .../HttpLogging/src/HttpLoggingOptions.cs | 21 ++ .../src/HttpLoggingServicesExtensions.cs | 53 +++ .../HttpLogging/src/HttpRequestLog.cs | 50 +++ .../HttpLogging/src/HttpResponseLog.cs | 50 +++ .../HttpLogging/src/LoggerEventIds.cs | 13 + .../Microsoft.AspNetCore.HttpLogging.csproj | 4 +- .../src/PublicAPI.Shipped.txt | 0 .../HttpLogging/src/PublicAPI.Unshipped.txt | 6 + .../src/ResponseBufferingStream.cs | 315 ++++++++++++++++++ .../test/HstsMiddlewareTests.cs | 0 .../test/HttpsPolicyTests.cs | 0 .../test/HttpsRedirectionMiddlewareTests.cs | 0 ...rosoft.AspNetCore.HttpsPolicy.Tests.csproj | 0 .../src/HttpLoggingBuilderExtensions.cs | 40 --- .../HttpLogging2/src/HttpLoggingExtensions.cs | 59 ---- .../HttpLogging2/src/HttpLoggingMiddleware.cs | 174 ---------- .../HttpLogging2/src/HttpLoggingOptions.cs | 29 -- .../src/HttpLoggingServicesExtensions.cs | 35 -- .../HttpLogging2/src/PublicAPI.Unshipped.txt | 1 - .../src/HttpsRedirectionOptions.cs | 2 +- 29 files changed, 890 insertions(+), 341 deletions(-) create mode 100644 src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj create mode 100644 src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs create mode 100644 src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json create mode 100644 src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs create mode 100644 src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json create mode 100644 src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json create mode 100644 src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs create mode 100644 src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs create mode 100644 src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs create mode 100644 src/Middleware/HttpLogging/src/HttpLoggingOptions.cs create mode 100644 src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs create mode 100644 src/Middleware/HttpLogging/src/HttpRequestLog.cs create mode 100644 src/Middleware/HttpLogging/src/HttpResponseLog.cs create mode 100644 src/Middleware/HttpLogging/src/LoggerEventIds.cs rename src/Middleware/{HttpLogging2 => HttpLogging}/src/Microsoft.AspNetCore.HttpLogging.csproj (81%) rename src/Middleware/{HttpLogging2 => HttpLogging}/src/PublicAPI.Shipped.txt (100%) create mode 100644 src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt create mode 100644 src/Middleware/HttpLogging/src/ResponseBufferingStream.cs rename src/Middleware/{HttpLogging2 => HttpLogging}/test/HstsMiddlewareTests.cs (100%) rename src/Middleware/{HttpLogging2 => HttpLogging}/test/HttpsPolicyTests.cs (100%) rename src/Middleware/{HttpLogging2 => HttpLogging}/test/HttpsRedirectionMiddlewareTests.cs (100%) rename src/Middleware/{HttpLogging2 => HttpLogging}/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj (100%) delete mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs delete mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs delete mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs delete mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs delete mode 100644 src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs delete mode 100644 src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj new file mode 100644 index 000000000000..971dcbc6f457 --- /dev/null +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj @@ -0,0 +1,14 @@ + + + + $(DefaultNetCoreTargetFramework) + + + + + + + + + + diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs new file mode 100644 index 000000000000..3eeae05dbf69 --- /dev/null +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs @@ -0,0 +1,26 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace HttpLogging.Sample +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json new file mode 100644 index 000000000000..2feff9ff305c --- /dev/null +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json @@ -0,0 +1,28 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:63240", + "sslPort": 0 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "HttpLogging.Sample": { + "commandName": "Project", + "dotnetRunMessages": "true", + "launchBrowser": true, + "applicationUrl": "http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs new file mode 100644 index 000000000000..a251380910c0 --- /dev/null +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs @@ -0,0 +1,35 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace HttpLogging.Sample +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Hello World!"); + }); + }); + } + } +} diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json new file mode 100644 index 000000000000..8983e0fc1c5e --- /dev/null +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json new file mode 100644 index 000000000000..d9d9a9bff6fd --- /dev/null +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs new file mode 100644 index 000000000000..ae00cce9d466 --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.HttpLogging; + +namespace Microsoft.AspNetCore.Builder +{ + /// + /// Extension methods for the HttpLogging middleware. + /// + public static class HttpLoggingBuilderExtensions + { + /// + /// Adds middleware for HTTP Logging. + /// + /// The instance this method extends. + /// The for HttpLogging. + public static IApplicationBuilder UseHttpLogging(this IApplicationBuilder app) + { + if (app == null) + { + throw new ArgumentNullException(nameof(app)); + } + + app.UseMiddleware(); + return app; + } + } +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs new file mode 100644 index 000000000000..6931964a6496 --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal static class HttpLoggingExtensions + { + static HttpLoggingExtensions() + { + } + } +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs new file mode 100644 index 000000000000..d7231c2875bc --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -0,0 +1,212 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.AspNetCore.HttpLogging +{ + /// + /// Middleware that logs HTTP requests and HTTP responses. + /// + public class HttpLoggingMiddleware + { + private readonly RequestDelegate _next; + + private readonly IConfiguration _config; + private readonly ILogger _logger; + private HttpLoggingOptions _options; + private const int PipeThreshold = 32 * 1024; + + /// + /// Initializes . + /// + /// + /// + /// + /// + public HttpLoggingMiddleware(RequestDelegate next, IOptions options, IConfiguration config, ILoggerFactory loggerFactory) + + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + _options = options.Value; + _logger = loggerFactory.CreateLogger(); + } + + /// + /// Invokes the . + /// + /// + /// + public async Task Invoke(HttpContext context) + { + if (!_options.DisableRequestLogging) + { + var list = new List>(); + + var request = context.Request; + list.Add(new KeyValuePair(nameof(request.Protocol), request.Protocol)); + list.Add(new KeyValuePair(nameof(request.Method), request.Method)); + list.Add(new KeyValuePair(nameof(request.ContentType), request.ContentType)); + list.Add(new KeyValuePair(nameof(request.ContentLength), request.ContentLength)); + list.Add(new KeyValuePair(nameof(request.Scheme), request.Scheme)); + list.Add(new KeyValuePair(nameof(request.Host), request.Host.Value)); + list.Add(new KeyValuePair(nameof(request.PathBase), request.PathBase.Value)); + list.Add(new KeyValuePair(nameof(request.Path), request.Path.Value)); + list.Add(new KeyValuePair(nameof(request.QueryString), request.QueryString.Value)); + + // Would hope for this to be nested somehow? Scope? + foreach (var header in FilterHeaders(request.Headers)) + { + list.Add(header); + } + + // reading the request body always seems expensive. + // TODO do we want string here? Other middleware writes to utf8jsonwriter directly. + var body = await ReadRequestBody(request, context.RequestAborted); + + list.Add(new KeyValuePair(nameof(request.Body), body)); + + // TODO add and remove things from log. + + var httpRequestLog = new HttpRequestLog(list); + _logger.Log(LogLevel.Information, + eventId: LoggerEventIds.RequestLog, + state: httpRequestLog, + exception: null, + formatter: HttpRequestLog.Callback); + } + + if (_options.DisableResponseLogging) + { + // Short circuit and don't replace response body. + await _next(context).ConfigureAwait(false); + return; + } + + var response = context.Response; + var originalBody = response.Body; + + // TODO pool memory streams. + + var originalBodyFeature = context.Features.Get()!; + var bufferingStream = new ResponseBufferingStream(originalBodyFeature, memoryStream, _options.ResponseBodyLogLimit); + response.Body = bufferingStream; + + try + { + await _next(context).ConfigureAwait(false); + var list = new List>(); + + // TODO elapsed milliseconds? + list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); + list.Add(new KeyValuePair(nameof(response.ContentType), response.ContentType)); + list.Add(new KeyValuePair(nameof(response.ContentLength), response.ContentLength)); + + var httpRequestLog = new HttpResponseLog(list); + foreach (var header in FilterHeaders(response.Headers)) + { + list.Add(header); + } + } + finally + { + context.Features.Set(originalBodyFeature); + } + } + + private IEnumerable> FilterHeaders(IHeaderDictionary headers) + { + foreach (var (key, value) in headers) + { + //if (_options.Filtering == HttpHeaderFiltering.OnlyListed && !_options.FilteringSet.Contains(key)) + //{ + // // Key is not among the "only listed" headers. + // continue; + //} + + //if (_options.Filtering == HttpHeaderFiltering.AllButListed && _options.FilteringSet.Contains(key)) + //{ + // // Key is among "all but listed" headers. + // continue; + //} + + //if (_options.Redact && _options.Redactors.TryGetValue(key, out var redactor)) + //{ + // yield return (key, redactor(value.ToString())); + // continue; + //} + + yield return new KeyValuePair(key, value.ToString()); + } + } + + private async Task ReadRequestBody(HttpRequest request, CancellationToken token) + { + using var joinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(); + joinedTokenSource.CancelAfter(_options.ReadTimeout); + var limit = _options.RequestBodyLogLimit; + if (limit <= PipeThreshold) + { + try + { + while (true) + { + var result = await request.BodyReader.ReadAsync(joinedTokenSource.Token); + if (!result.IsCompleted && result.Buffer.Length <= limit) + { + // Need more data. + request.BodyReader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + continue; + } + + var res = Encoding.UTF8.GetString(result.Buffer.Slice(0, result.Buffer.Length > limit ? limit : result.Buffer.Length)); + request.BodyReader.AdvanceTo(result.Buffer.Start, result.Buffer.End); + return res; + } + } + catch (OperationCanceledException) + { + // Token source hides triggering token (https://github.com/dotnet/runtime/issues/22172) + if (!token.IsCancellationRequested && joinedTokenSource.Token.IsCancellationRequested) + { + // TODO should this be empty instead? + return "[Cancelled]"; + } + + throw; + } + } + else + { + // TODO determine buffering limits here. + request.EnableBuffering(); + + // Read here. + + if (request.Body.CanSeek) + { + _ = request.Body.Seek(0, SeekOrigin.Begin); + } + } + + return ""; + } + } +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs new file mode 100644 index 000000000000..ea0a7a6ab9f8 --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.Http; + +namespace Microsoft.AspNetCore.HttpLogging +{ + public class HttpLoggingOptions + { + // TODO make these reloadable. + public bool DisableRequestLogging { get; set; } = false; + public bool DisableResponseLogging { get; set; } = false; + + public int RequestBodyLogLimit { get; set; } = 32 * 1024; // 32KB + public TimeSpan ReadTimeout { get; set; } = TimeSpan.FromSeconds(1); + + public int ResponseBodyLogLimit { get; set; } = 32 * 1024; // 32KB + + } +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs new file mode 100644 index 000000000000..0402bc550c7e --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.HttpLogging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace Microsoft.AspNetCore.Builder +{ + /// + /// Extension methods for the HttpLogging middleware. + /// + public static class HttpLoggingServicesExtensions + { + /// + /// Adds HTTP Logging services. + /// + /// The for adding services. + /// + public static IServiceCollection AddHttpLogging(this IServiceCollection services) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + + services.TryAddSingleton(); + return services; + } + + /// + /// Adds HTTP Logging services. + /// + /// The for adding services. + /// A delegate to configure the . + /// + public static IServiceCollection AddHttpLogging(this IServiceCollection services, Action configureOptions) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + if (configureOptions == null) + { + throw new ArgumentNullException(nameof(configureOptions)); + } + + services.Configure(configureOptions); + return services; + } + } +} diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs new file mode 100644 index 000000000000..03bf3a7dbc77 --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal class HttpRequestLog : IReadOnlyList> + { + private readonly List> _keyValues; + private string? _cachedToString; + + internal static readonly Func Callback = (state, exception) => ((HttpRequestLog)state).ToString(); + + public HttpRequestLog(List> keyValues) + { + _keyValues = keyValues; + } + + public KeyValuePair this[int index] => _keyValues[index]; + + public int Count => _keyValues.Count; + + public IEnumerator> GetEnumerator() + { + for (var i = 0; i < Count; i++) + { + yield return _keyValues[i]; + } + } + + public override string ToString() + { + if (_cachedToString == null) + { + // TODO new line separated list? + _cachedToString = ""; + } + + return _cachedToString; + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs new file mode 100644 index 000000000000..aa50e1bb496f --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal class HttpResponseLog : IReadOnlyList> + { + private readonly List> _keyValues; + private string? _cachedToString; + + internal static readonly Func Callback = (state, exception) => ((HttpResponseLog)state).ToString(); + + public HttpResponseLog(List> keyValues) + { + _keyValues = keyValues; + } + + public KeyValuePair this[int index] => _keyValues[index]; + + public int Count => _keyValues.Count; + + public IEnumerator> GetEnumerator() + { + for (var i = 0; i < Count; i++) + { + yield return _keyValues[i]; + } + } + + public override string ToString() + { + if (_cachedToString == null) + { + // TODO new line separated list? + _cachedToString = ""; + } + + return _cachedToString; + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/src/Middleware/HttpLogging/src/LoggerEventIds.cs b/src/Middleware/HttpLogging/src/LoggerEventIds.cs new file mode 100644 index 000000000000..83780d4b5599 --- /dev/null +++ b/src/Middleware/HttpLogging/src/LoggerEventIds.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal static class LoggerEventIds + { + public static readonly EventId RequestLog = new EventId(1, "RequestLog"); + public static readonly EventId ResponseLog = new EventId(2, "ResponseLog"); + } +} diff --git a/src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj similarity index 81% rename from src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj rename to src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index c14e35c11fda..fa0f5f6b7fc6 100644 --- a/src/Middleware/HttpLogging2/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -2,12 +2,12 @@ - ASP.NET Core basic middleware for supporting HTTPS Redirection and HTTP Strict-Transport-Security. + ASP.NET Core middleware for logging HTTP requests and responses. $(DefaultNetCoreTargetFramework) true true - aspnetcore;https;hsts + aspnetcore;logging;http false enable diff --git a/src/Middleware/HttpLogging2/src/PublicAPI.Shipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Shipped.txt similarity index 100% rename from src/Middleware/HttpLogging2/src/PublicAPI.Shipped.txt rename to src/Middleware/HttpLogging/src/PublicAPI.Shipped.txt diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt new file mode 100644 index 000000000000..d0af8d8b4f5c --- /dev/null +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -0,0 +1,6 @@ +#nullable enable +Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions +Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware +Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.Invoke(Microsoft.AspNetCore.Http.HttpContext! context) -> System.Threading.Tasks.Task! +static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! +~Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.HttpLoggingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate! next, Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Configuration.IConfiguration! config, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs new file mode 100644 index 000000000000..5e374ef86420 --- /dev/null +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -0,0 +1,315 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature + { + private readonly IHttpResponseBodyFeature _innerBodyFeature; + private readonly Stream _innerStream; + private readonly int _limit; + private PipeWriter? _pipeAdapter; + + private readonly int _minimumSegmentSize; + private int _bytesWritten; + private List? _completedSegments; + private byte[]? _currentSegment; + private int _position; + + internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit) + { + _innerBodyFeature = innerBodyFeature; + _innerStream = innerBodyFeature.Stream; + _limit = limit; + _minimumSegmentSize = 4096; + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => _innerStream.CanWrite; + + public override long Length + { + get { throw new NotSupportedException(); } + } + + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + public Stream Stream => this; + + public PipeWriter Writer + { + get + { + if (_pipeAdapter == null) + { + _pipeAdapter = PipeWriter.Create(Stream, new StreamPipeWriterOptions(leaveOpen: true)); + } + + return _pipeAdapter; + } + } + + public ReadOnlyMemory GetBuffer() + { + return _buffer.ToArray(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Flush() + { + _innerStream.Flush(); + } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + return _innerStream.FlushAsync(cancellationToken); + } + + public override void Write(byte[] buffer, int offset, int count) + { + var remaining = _limit - (int)_buffer.Length; + if (remaining > 0) + { + _buffer.Write(buffer, offset, Math.Min(count, remaining)); + } + _innerStream.Write(buffer, offset, count); + } + + public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) + { + var tcs = new TaskCompletionSource(state: state, TaskCreationOptions.RunContinuationsAsynchronously); + InternalWriteAsync(buffer, offset, count, callback, tcs); + return tcs.Task; + } + + private async void InternalWriteAsync(byte[] buffer, int offset, int count, AsyncCallback? callback, TaskCompletionSource tcs) + { + try + { + await WriteAsync(buffer, offset, count); + tcs.TrySetResult(); + } + catch (Exception ex) + { + tcs.TrySetException(ex); + } + + if (callback != null) + { + // Offload callbacks to avoid stack dives on sync completions. + var ignored = Task.Run(() => + { + try + { + callback(tcs.Task); + } + catch (Exception) + { + // Suppress exceptions on background threads. + } + }); + } + } + + public override void EndWrite(IAsyncResult asyncResult) + { + if (asyncResult == null) + { + throw new ArgumentNullException(nameof(asyncResult)); + } + + var task = (Task)asyncResult; + task.GetAwaiter().GetResult(); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var remaining = _limit - (int)_buffer.Length; + if (remaining > 0) + { + await _buffer.WriteAsync(buffer, offset, Math.Min(count, remaining)); + } + + await _innerStream.WriteAsync(buffer, offset, count, cancellationToken); + } + + public void DisableBuffering() + { + _innerBodyFeature.DisableBuffering(); + } + + public Task SendFileAsync(string path, long offset, long? count, CancellationToken cancellation) + { + return _innerBodyFeature.SendFileAsync(path, offset, count, cancellation); + } + + public Task StartAsync(CancellationToken token = default) + { + return _innerBodyFeature.StartAsync(token); + } + + public async Task CompleteAsync() + { + await _innerBodyFeature.CompleteAsync(); + } + + public void Advance(int count) + { + _bytesWritten += count; + _position += count; + } + + public Memory GetMemory(int sizeHint = 0) + { + EnsureCapacity(sizeHint); + + return _currentSegment.AsMemory(_position, _currentSegment.Length - _position); + } + + public Span GetSpan(int sizeHint = 0) + { + EnsureCapacity(sizeHint); + + return _currentSegment.AsSpan(_position, _currentSegment.Length - _position); + } + public override void WriteByte(byte value) + { + if (_currentSegment != null && (uint)_position < (uint)_currentSegment.Length) + { + _currentSegment[_position] = value; + } + else + { + AddSegment(); + _currentSegment[0] = value; + } + + _position++; + _bytesWritten++; + } + + public override void Write(byte[] buffer, int offset, int count) + { + var position = _position; + if (_currentSegment != null && position < _currentSegment.Length - count) + { + Buffer.BlockCopy(buffer, offset, _currentSegment, position, count); + + _position = position + count; + _bytesWritten += count; + } + else + { + BuffersExtensions.Write(this, buffer.AsSpan(offset, count)); + } + } + +#if NETCOREAPP + public override void Write(ReadOnlySpan span) + { + if (_currentSegment != null && span.TryCopyTo(_currentSegment.AsSpan(_position))) + { + _position += span.Length; + _bytesWritten += span.Length; + } + else + { + BuffersExtensions.Write(this, span); + } + } + + [MemberNotNull(nameof(_currentSegment))] + private void EnsureCapacity(int sizeHint) + { + // This does the Right Thing. It only subtracts _position from the current segment length if it's non-null. + // If _currentSegment is null, it returns 0. + var remainingSize = _currentSegment?.Length - _position ?? 0; + + // If the sizeHint is 0, any capacity will do + // Otherwise, the buffer must have enough space for the entire size hint, or we need to add a segment. + if ((sizeHint == 0 && remainingSize > 0) || (sizeHint > 0 && remainingSize >= sizeHint)) + { + // We have capacity in the current segment +#pragma warning disable CS8774 // Member must have a non-null value when exiting. + return; +#pragma warning restore CS8774 // Member must have a non-null value when exiting. + } + + AddSegment(sizeHint); + } + + [MemberNotNull(nameof(_currentSegment))] + private void AddSegment(int sizeHint = 0) + { + if (_currentSegment != null) + { + // We're adding a segment to the list + if (_completedSegments == null) + { + _completedSegments = new List(); + } + + // Position might be less than the segment length if there wasn't enough space to satisfy the sizeHint when + // GetMemory was called. In that case we'll take the current segment and call it "completed", but need to + // ignore any empty space in it. + _completedSegments.Add(new CompletedBuffer(_currentSegment, _position)); + } + + // Get a new buffer using the minimum segment size, unless the size hint is larger than a single segment. + _currentSegment = ArrayPool.Shared.Rent(Math.Max(_minimumSegmentSize, sizeHint)); + _position = 0; + } + + /// + /// Holds a byte[] from the pool and a size value. Basically a Memory but guaranteed to be backed by an ArrayPool byte[], so that we know we can return it. + /// + private readonly struct CompletedBuffer + { + public byte[] Buffer { get; } + public int Length { get; } + + public ReadOnlySpan Span => Buffer.AsSpan(0, Length); + + public CompletedBuffer(byte[] buffer, int length) + { + Buffer = buffer; + Length = length; + } + + public void Return() + { + ArrayPool.Shared.Return(Buffer); + } + } + } +} diff --git a/src/Middleware/HttpLogging2/test/HstsMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HstsMiddlewareTests.cs similarity index 100% rename from src/Middleware/HttpLogging2/test/HstsMiddlewareTests.cs rename to src/Middleware/HttpLogging/test/HstsMiddlewareTests.cs diff --git a/src/Middleware/HttpLogging2/test/HttpsPolicyTests.cs b/src/Middleware/HttpLogging/test/HttpsPolicyTests.cs similarity index 100% rename from src/Middleware/HttpLogging2/test/HttpsPolicyTests.cs rename to src/Middleware/HttpLogging/test/HttpsPolicyTests.cs diff --git a/src/Middleware/HttpLogging2/test/HttpsRedirectionMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpsRedirectionMiddlewareTests.cs similarity index 100% rename from src/Middleware/HttpLogging2/test/HttpsRedirectionMiddlewareTests.cs rename to src/Middleware/HttpLogging/test/HttpsRedirectionMiddlewareTests.cs diff --git a/src/Middleware/HttpLogging2/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj b/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj similarity index 100% rename from src/Middleware/HttpLogging2/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj rename to src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs deleted file mode 100644 index c6f804763fff..000000000000 --- a/src/Middleware/HttpLogging2/src/HttpLoggingBuilderExtensions.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.HttpsPolicy; - -namespace Microsoft.AspNetCore.Builder -{ - /// - /// Extension methods for the HttpsRedirection middleware. - /// - public static class HttpsPolicyBuilderExtensions - { - /// - /// Adds middleware for redirecting HTTP Requests to HTTPS. - /// - /// The instance this method extends. - /// The for HttpsRedirection. - public static IApplicationBuilder UseHttpsRedirection(this IApplicationBuilder app) - { - if (app == null) - { - throw new ArgumentNullException(nameof(app)); - } - - var serverAddressFeature = app.ServerFeatures.Get(); - if (serverAddressFeature != null) - { - app.UseMiddleware(serverAddressFeature); - } - else - { - app.UseMiddleware(); - } - return app; - } - } -} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs deleted file mode 100644 index f3b00416f566..000000000000 --- a/src/Middleware/HttpLogging2/src/HttpLoggingExtensions.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using Microsoft.Extensions.Logging; - -namespace Microsoft.AspNetCore.HttpsPolicy -{ - internal static class HttpsLoggingExtensions - { - private static readonly Action _redirectingToHttps; - private static readonly Action _portLoadedFromConfig; - private static readonly Action _failedToDeterminePort; - private static readonly Action _portFromServer; - - static HttpsLoggingExtensions() - { - _redirectingToHttps = LoggerMessage.Define( - LogLevel.Debug, - new EventId(1, "RedirectingToHttps"), - "Redirecting to '{redirect}'."); - - _portLoadedFromConfig = LoggerMessage.Define( - LogLevel.Debug, - new EventId(2, "PortLoadedFromConfig"), - "Https port '{port}' loaded from configuration."); - - _failedToDeterminePort = LoggerMessage.Define( - LogLevel.Warning, - new EventId(3, "FailedToDeterminePort"), - "Failed to determine the https port for redirect."); - - _portFromServer = LoggerMessage.Define( - LogLevel.Debug, - new EventId(5, "PortFromServer"), - "Https port '{httpsPort}' discovered from server endpoints."); - } - - public static void RedirectingToHttps(this ILogger logger, string redirect) - { - _redirectingToHttps(logger, redirect, null); - } - - public static void PortLoadedFromConfig(this ILogger logger, int port) - { - _portLoadedFromConfig(logger, port, null); - } - - public static void FailedToDeterminePort(this ILogger logger) - { - _failedToDeterminePort(logger, null); - } - - public static void PortFromServer(this ILogger logger, int port) - { - _portFromServer(logger, port, null); - } - } -} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs deleted file mode 100644 index 215507912f3e..000000000000 --- a/src/Middleware/HttpLogging2/src/HttpLoggingMiddleware.cs +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Extensions; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Net.Http.Headers; - -namespace Microsoft.AspNetCore.HttpsPolicy -{ - /// - /// Middleware that redirects non-HTTPS requests to an HTTPS URL. - /// - public class HttpsRedirectionMiddleware - { - private const int PortNotFound = -1; - - private readonly RequestDelegate _next; - private readonly Lazy _httpsPort; - private readonly int _statusCode; - - private readonly IServerAddressesFeature? _serverAddressesFeature; - private readonly IConfiguration _config; - private readonly ILogger _logger; - - /// - /// Initializes . - /// - /// - /// - /// - /// - public HttpsRedirectionMiddleware(RequestDelegate next, IOptions options, IConfiguration config, ILoggerFactory loggerFactory) - - { - _next = next ?? throw new ArgumentNullException(nameof(next)); - _config = config ?? throw new ArgumentNullException(nameof(config)); - - if (options == null) - { - throw new ArgumentNullException(nameof(options)); - } - var httpsRedirectionOptions = options.Value; - if (httpsRedirectionOptions.HttpsPort.HasValue) - { - _httpsPort = new Lazy(httpsRedirectionOptions.HttpsPort.Value); - } - else - { - _httpsPort = new Lazy(TryGetHttpsPort); - } - _statusCode = httpsRedirectionOptions.RedirectStatusCode; - _logger = loggerFactory.CreateLogger(); - } - - /// - /// Initializes . - /// - /// - /// - /// - /// - /// - public HttpsRedirectionMiddleware(RequestDelegate next, IOptions options, IConfiguration config, ILoggerFactory loggerFactory, - IServerAddressesFeature serverAddressesFeature) - : this(next, options, config, loggerFactory) - { - _serverAddressesFeature = serverAddressesFeature ?? throw new ArgumentNullException(nameof(serverAddressesFeature)); - } - - /// - /// Invokes the HttpsRedirectionMiddleware. - /// - /// - /// - public Task Invoke(HttpContext context) - { - if (context.Request.IsHttps) - { - return _next(context); - } - - var port = _httpsPort.Value; - if (port == PortNotFound) - { - return _next(context); - } - - var host = context.Request.Host; - if (port != 443) - { - host = new HostString(host.Host, port); - } - else - { - host = new HostString(host.Host); - } - - var request = context.Request; - var redirectUrl = UriHelper.BuildAbsolute( - "https", - host, - request.PathBase, - request.Path, - request.QueryString); - - context.Response.StatusCode = _statusCode; - context.Response.Headers[HeaderNames.Location] = redirectUrl; - - _logger.RedirectingToHttps(redirectUrl); - - return Task.CompletedTask; - } - - // Returns PortNotFound (-1) if we were unable to determine the port. - private int TryGetHttpsPort() - { - // The IServerAddressesFeature will not be ready until the middleware is Invoked, - // Order for finding the HTTPS port: - // 1. Set in the HttpsRedirectionOptions - // 2. HTTPS_PORT environment variable - // 3. IServerAddressesFeature - // 4. Fail if not sets - - var nullablePort = _config.GetValue("HTTPS_PORT") ?? _config.GetValue("ANCM_HTTPS_PORT"); - if (nullablePort.HasValue) - { - var port = nullablePort.Value; - _logger.PortLoadedFromConfig(port); - return port; - } - - if (_serverAddressesFeature == null) - { - _logger.FailedToDeterminePort(); - return PortNotFound; - } - - foreach (var address in _serverAddressesFeature.Addresses) - { - var bindingAddress = BindingAddress.Parse(address); - if (bindingAddress.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) - { - // If we find multiple different https ports specified, throw - if (nullablePort.HasValue && nullablePort != bindingAddress.Port) - { - throw new InvalidOperationException( - "Cannot determine the https port from IServerAddressesFeature, multiple values were found. " + - "Set the desired port explicitly on HttpsRedirectionOptions.HttpsPort."); - } - else - { - nullablePort = bindingAddress.Port; - } - } - } - - if (nullablePort.HasValue) - { - var port = nullablePort.Value; - _logger.PortFromServer(port); - return port; - } - - _logger.FailedToDeterminePort(); - return PortNotFound; - } - } -} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs deleted file mode 100644 index 0e1c625bd7dc..000000000000 --- a/src/Middleware/HttpLogging2/src/HttpLoggingOptions.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using Microsoft.AspNetCore.Http; - -namespace Microsoft.AspNetCore.HttpsPolicy -{ - /// - /// Options for the HTTPS Redirection Middleware. - /// - public class HttpsRedirectionOptions - { - /// - /// The status code used for the redirect response. The default is 307. - /// - public int RedirectStatusCode { get; set; } = StatusCodes.Status307TemporaryRedirect; - - /// - /// The HTTPS port to be added to the redirected URL. - /// - /// - /// If the HttpsPort is not set, we will try to get the HttpsPort from the following: - /// 1. HTTPS_PORT environment variable - /// 2. IServerAddressesFeature - /// If that fails then the middleware will log a warning and turn off. - /// - public int? HttpsPort { get; set; } - } -} diff --git a/src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs deleted file mode 100644 index 24c9e114e8b9..000000000000 --- a/src/Middleware/HttpLogging2/src/HttpLoggingServicesExtensions.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using Microsoft.AspNetCore.HttpsPolicy; -using Microsoft.Extensions.DependencyInjection; - -namespace Microsoft.AspNetCore.Builder -{ - /// - /// Extension methods for the HttpsRedirection middleware. - /// - public static class HttpsRedirectionServicesExtensions - { - /// - /// Adds HTTPS redirection services. - /// - /// The for adding services. - /// A delegate to configure the . - /// - public static IServiceCollection AddHttpsRedirection(this IServiceCollection services, Action configureOptions) - { - if (services == null) - { - throw new ArgumentNullException(nameof(services)); - } - if (configureOptions == null) - { - throw new ArgumentNullException(nameof(configureOptions)); - } - services.Configure(configureOptions); - return services; - } - } -} diff --git a/src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt deleted file mode 100644 index 7dc5c58110bf..000000000000 --- a/src/Middleware/HttpLogging2/src/PublicAPI.Unshipped.txt +++ /dev/null @@ -1 +0,0 @@ -#nullable enable diff --git a/src/Middleware/HttpsPolicy/src/HttpsRedirectionOptions.cs b/src/Middleware/HttpsPolicy/src/HttpsRedirectionOptions.cs index 0e1c625bd7dc..632b40f709d5 100644 --- a/src/Middleware/HttpsPolicy/src/HttpsRedirectionOptions.cs +++ b/src/Middleware/HttpsPolicy/src/HttpsRedirectionOptions.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Microsoft.AspNetCore.Http; From cef0a0e7cbecc308afba16558b8cb3398ac387bf Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 9 Apr 2021 10:03:06 -0700 Subject: [PATCH 03/45] Logging --- AspNetCore.sln | 48 ++ ...MiddlewareTests.cs => HttpLoggingTests.cs} | 0 .../HttpLogging/test/HttpsPolicyTests.cs | 86 ---- .../test/HttpsRedirectionMiddlewareTests.cs | 462 ------------------ ...osoft.AspNetCore.HttpLogging.Tests.csproj} | 0 5 files changed, 48 insertions(+), 548 deletions(-) rename src/Middleware/HttpLogging/test/{HstsMiddlewareTests.cs => HttpLoggingTests.cs} (100%) delete mode 100644 src/Middleware/HttpLogging/test/HttpsPolicyTests.cs delete mode 100644 src/Middleware/HttpLogging/test/HttpsRedirectionMiddlewareTests.cs rename src/Middleware/HttpLogging/test/{Microsoft.AspNetCore.HttpsPolicy.Tests.csproj => Microsoft.AspNetCore.HttpLogging.Tests.csproj} (100%) diff --git a/AspNetCore.sln b/AspNetCore.sln index 81801038c953..9e5ec5932ba1 100644 --- a/AspNetCore.sln +++ b/AspNetCore.sln @@ -1626,6 +1626,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.AspNetCore.SpaSer EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinimalSample", "src\Http\samples\MinimalSample\MinimalSample.csproj", "{9647D8B7-4616-4E05-B258-BAD5CAEEDD38}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpLogging", "HttpLogging", "{022B4B80-E813-4256-8034-11A68146F4EF}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.HttpLogging", "src\Middleware\HttpLogging\src\Microsoft.AspNetCore.HttpLogging.csproj", "{FF413F1C-A998-4FA2-823F-52AC0916B35C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.HttpLogging.Tests", "src\Middleware\HttpLogging\test\Microsoft.AspNetCore.HttpLogging.Tests.csproj", "{3A1EC883-EF9C-43E8-95E5-6B527428867B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HttpLogging.Sample", "src\Middleware\HttpLogging\samples\HttpLogging.Sample\HttpLogging.Sample.csproj", "{908B2263-B58B-4261-A125-B5F2DFF92799}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -7709,6 +7717,42 @@ Global {9647D8B7-4616-4E05-B258-BAD5CAEEDD38}.Release|x64.Build.0 = Release|Any CPU {9647D8B7-4616-4E05-B258-BAD5CAEEDD38}.Release|x86.ActiveCfg = Release|Any CPU {9647D8B7-4616-4E05-B258-BAD5CAEEDD38}.Release|x86.Build.0 = Release|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Debug|x64.ActiveCfg = Debug|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Debug|x64.Build.0 = Debug|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Debug|x86.ActiveCfg = Debug|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Debug|x86.Build.0 = Debug|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Release|Any CPU.Build.0 = Release|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Release|x64.ActiveCfg = Release|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Release|x64.Build.0 = Release|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Release|x86.ActiveCfg = Release|Any CPU + {FF413F1C-A998-4FA2-823F-52AC0916B35C}.Release|x86.Build.0 = Release|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Debug|x64.ActiveCfg = Debug|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Debug|x64.Build.0 = Debug|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Debug|x86.ActiveCfg = Debug|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Debug|x86.Build.0 = Debug|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Release|Any CPU.Build.0 = Release|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Release|x64.ActiveCfg = Release|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Release|x64.Build.0 = Release|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Release|x86.ActiveCfg = Release|Any CPU + {3A1EC883-EF9C-43E8-95E5-6B527428867B}.Release|x86.Build.0 = Release|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Debug|Any CPU.Build.0 = Debug|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Debug|x64.ActiveCfg = Debug|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Debug|x64.Build.0 = Debug|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Debug|x86.ActiveCfg = Debug|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Debug|x86.Build.0 = Debug|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Release|Any CPU.ActiveCfg = Release|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Release|Any CPU.Build.0 = Release|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Release|x64.ActiveCfg = Release|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Release|x64.Build.0 = Release|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Release|x86.ActiveCfg = Release|Any CPU + {908B2263-B58B-4261-A125-B5F2DFF92799}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -8514,6 +8558,10 @@ Global {DF4637DA-5F07-4903-8461-4E2DAB235F3C} = {7F99E967-3DC1-4198-9D55-47CD9471D0B6} {AAB50C64-39AA-4AED-8E9C-50D68E7751AD} = {7F99E967-3DC1-4198-9D55-47CD9471D0B6} {9647D8B7-4616-4E05-B258-BAD5CAEEDD38} = {EB5E294B-9ED5-43BF-AFA9-1CD2327F3DC1} + {022B4B80-E813-4256-8034-11A68146F4EF} = {E5963C9F-20A6-4385-B364-814D2581FADF} + {FF413F1C-A998-4FA2-823F-52AC0916B35C} = {022B4B80-E813-4256-8034-11A68146F4EF} + {3A1EC883-EF9C-43E8-95E5-6B527428867B} = {022B4B80-E813-4256-8034-11A68146F4EF} + {908B2263-B58B-4261-A125-B5F2DFF92799} = {022B4B80-E813-4256-8034-11A68146F4EF} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3E8720B3-DBDD-498C-B383-2CC32A054E8F} diff --git a/src/Middleware/HttpLogging/test/HstsMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingTests.cs similarity index 100% rename from src/Middleware/HttpLogging/test/HstsMiddlewareTests.cs rename to src/Middleware/HttpLogging/test/HttpLoggingTests.cs diff --git a/src/Middleware/HttpLogging/test/HttpsPolicyTests.cs b/src/Middleware/HttpLogging/test/HttpsPolicyTests.cs deleted file mode 100644 index d246c6255f01..000000000000 --- a/src/Middleware/HttpLogging/test/HttpsPolicyTests.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.TestHost; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Net.Http.Headers; -using Xunit; - -namespace Microsoft.AspNetCore.HttpsPolicy.Tests -{ - public class HttpsPolicyTests - { - [Theory] - [InlineData(302, 443, 2592000, false, false, "max-age=2592000", "https://localhost/")] - [InlineData(301, 5050, 2592000, false, false, "max-age=2592000", "https://localhost:5050/")] - [InlineData(301, 443, 2592000, false, false, "max-age=2592000", "https://localhost/")] - [InlineData(301, 443, 2592000, true, false, "max-age=2592000; includeSubDomains", "https://localhost/")] - [InlineData(301, 443, 2592000, false, true, "max-age=2592000; preload", "https://localhost/")] - [InlineData(301, 443, 2592000, true, true, "max-age=2592000; includeSubDomains; preload", "https://localhost/")] - [InlineData(302, 5050, 2592000, true, true, "max-age=2592000; includeSubDomains; preload", "https://localhost:5050/")] - public async Task SetsBothHstsAndHttpsRedirection_RedirectOnFirstRequest_HstsOnSecondRequest(int statusCode, int? tlsPort, int maxAge, bool includeSubDomains, bool preload, string expectedHstsHeader, string expectedUrl) - { - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.Configure(options => - { - options.RedirectStatusCode = statusCode; - options.HttpsPort = tlsPort; - }); - services.Configure(options => - { - options.IncludeSubDomains = includeSubDomains; - options.MaxAge = TimeSpan.FromSeconds(maxAge); - options.Preload = preload; - options.ExcludedHosts.Clear(); // allowing localhost for testing - }); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - server.Features.Set(new ServerAddressesFeature()); - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(statusCode, (int)response.StatusCode); - Assert.Equal(expectedUrl, response.Headers.Location.ToString()); - - client = server.CreateClient(); - client.BaseAddress = new Uri(response.Headers.Location.ToString()); - request = new HttpRequestMessage(HttpMethod.Get, expectedUrl); - response = await client.SendAsync(request); - - Assert.Equal(expectedHstsHeader, response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); - } - } -} diff --git a/src/Middleware/HttpLogging/test/HttpsRedirectionMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpsRedirectionMiddlewareTests.cs deleted file mode 100644 index d4bd8fc7df2e..000000000000 --- a/src/Middleware/HttpLogging/test/HttpsRedirectionMiddlewareTests.cs +++ /dev/null @@ -1,462 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Hosting.Server; -using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.TestHost; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Testing; -using Xunit; - -namespace Microsoft.AspNetCore.HttpsPolicy.Tests -{ - public class HttpsRedirectionMiddlewareTests - { - [Fact] - public async Task SetOptions_NotEnabledByDefault() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Warning, message.LogLevel); - Assert.Equal("Failed to determine the https port for redirect.", message.State.ToString()); - } - - [Theory] - [InlineData(302, 5001, "https://localhost:5001/")] - [InlineData(307, 1, "https://localhost:1/")] - [InlineData(308, 3449, "https://localhost:3449/")] - [InlineData(301, 5050, "https://localhost:5050/")] - [InlineData(301, 443, "https://localhost/")] - public async Task SetOptions_SetStatusCodeHttpsPort(int statusCode, int? httpsPort, string expected) - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - services.Configure(options => - { - options.RedirectStatusCode = statusCode; - options.HttpsPort = httpsPort; - }); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(statusCode, (int)response.StatusCode); - Assert.Equal(expected, response.Headers.Location.ToString()); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal($"Redirecting to '{expected}'.", message.State.ToString()); - } - - [Theory] - [InlineData(302, 5001, "https://localhost:5001/")] - [InlineData(307, 1, "https://localhost:1/")] - [InlineData(308, 3449, "https://localhost:3449/")] - [InlineData(301, 5050, "https://localhost:5050/")] - [InlineData(301, 443, "https://localhost/")] - public async Task SetOptionsThroughHelperMethod_SetStatusCodeAndHttpsPort(int statusCode, int? httpsPort, string expectedUrl) - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - services.AddHttpsRedirection(options => - { - options.RedirectStatusCode = statusCode; - options.HttpsPort = httpsPort; - }); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(statusCode, (int)response.StatusCode); - Assert.Equal(expectedUrl, response.Headers.Location.ToString()); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal($"Redirecting to '{expectedUrl}'.", message.State.ToString()); - } - - [Theory] - [InlineData(null, null, "https://localhost:4444/", "https://localhost:4444/")] - [InlineData(null, null, "https://localhost:443/", "https://localhost/")] - [InlineData(null, null, "https://localhost/", "https://localhost/")] - [InlineData(null, "5000", "https://localhost:4444/", "https://localhost:5000/")] - [InlineData(null, "443", "https://localhost:4444/", "https://localhost/")] - [InlineData(443, "5000", "https://localhost:4444/", "https://localhost/")] - [InlineData(4000, "5000", "https://localhost:4444/", "https://localhost:4000/")] - [InlineData(5000, null, "https://localhost:4444/", "https://localhost:5000/")] - public async Task SetHttpsPortEnvironmentVariableAndServerFeature_ReturnsCorrectStatusCodeOnResponse( - int? optionsHttpsPort, string configHttpsPort, string serverAddressFeatureUrl, string expectedUrl) - { - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddHttpsRedirection(options => - { - options.HttpsPort = optionsHttpsPort; - }); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - - webHostBuilder.UseSetting("HTTPS_PORT", configHttpsPort); - }).Build(); - - var server = host.GetTestServer(); - server.Features.Set(new ServerAddressesFeature()); - if (serverAddressFeatureUrl != null) - { - server.Features.Get().Addresses.Add(serverAddressFeatureUrl); - } - - await host.StartAsync(); - - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(expectedUrl, response.Headers.Location.ToString()); - } - - [Fact] - public async Task SetServerAddressesFeature_SingleHttpsAddress_Success() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - var server = host.GetTestServer(); - server.Features.Set(new ServerAddressesFeature()); - - server.Features.Get().Addresses.Add("https://localhost:5050"); - await host.StartAsync(); - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal("https://localhost:5050/", response.Headers.Location.ToString()); - - var logMessages = sink.Writes.ToList(); - - Assert.Equal(2, logMessages.Count); - var message = logMessages.First(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal("Https port '5050' discovered from server endpoints.", message.State.ToString()); - - message = logMessages.Skip(1).First(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal("Redirecting to 'https://localhost:5050/'.", message.State.ToString()); - } - - [Fact] - public async Task SetServerAddressesFeature_MultipleHttpsAddresses_Throws() - { - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - var server = host.GetTestServer(); - server.Features.Set(new ServerAddressesFeature()); - - server.Features.Get().Addresses.Add("https://localhost:5050"); - server.Features.Get().Addresses.Add("https://localhost:5051"); - - await host.StartAsync(); - - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var ex = await Assert.ThrowsAsync(() => client.SendAsync(request)); - Assert.Equal("Cannot determine the https port from IServerAddressesFeature, multiple values were found. " + - "Set the desired port explicitly on HttpsRedirectionOptions.HttpsPort.", ex.Message); - } - - [Fact] - public async Task SetServerAddressesFeature_MultipleHttpsAddressesWithSamePort_Success() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - var server = host.GetTestServer(); - server.Features.Set(new ServerAddressesFeature()); - server.Features.Get().Addresses.Add("https://localhost:5050"); - server.Features.Get().Addresses.Add("https://example.com:5050"); - - await host.StartAsync(); - - var client = server.CreateClient(); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal("https://localhost:5050/", response.Headers.Location.ToString()); - - var logMessages = sink.Writes.ToList(); - - Assert.Equal(2, logMessages.Count); - var message = logMessages.First(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal("Https port '5050' discovered from server endpoints.", message.State.ToString()); - - message = logMessages.Skip(1).First(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal("Redirecting to 'https://localhost:5050/'.", message.State.ToString()); - } - - [Fact] - public async Task NoServerAddressFeature_DoesNotThrow_DoesNotRedirect() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - var response = await client.SendAsync(request); - Assert.Equal(200, (int)response.StatusCode); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.First(); - Assert.Equal(LogLevel.Warning, message.LogLevel); - Assert.Equal("Failed to determine the https port for redirect.", message.State.ToString()); - } - - [Fact] - public async Task SetNullAddressFeature_DoesNotThrow() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHttpsRedirection(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - var server = host.GetTestServer(); - server.Features.Set(null); - - await host.StartAsync(); - - var client = server.CreateClient(); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - var response = await client.SendAsync(request); - Assert.Equal(200, (int)response.StatusCode); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.First(); - Assert.Equal(LogLevel.Warning, message.LogLevel); - Assert.Equal("Failed to determine the https port for redirect.", message.State.ToString()); - } - } -} diff --git a/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj b/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpLogging.Tests.csproj similarity index 100% rename from src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpsPolicy.Tests.csproj rename to src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpLogging.Tests.csproj From 6c6f371e68203987501958dfbde4a49557213d31 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 12 Apr 2021 09:26:28 -0700 Subject: [PATCH 04/45] nit --- src/Middleware/HttpLogging/test/HttpLoggingTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingTests.cs index 232f2e474803..e97e3226fb62 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingTests.cs @@ -20,7 +20,7 @@ namespace Microsoft.AspNetCore.HttpsPolicy.Tests { - public class HstsMiddlewareTests + public class HttpLoggingTests { [Fact] public async Task SetOptionsWithDefault_SetsMaxAgeToCorrectValue() From cb167a3b954e39dea66a2c99a1c0a2fdfb735d0f Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 14 Apr 2021 14:12:17 -0700 Subject: [PATCH 05/45] Polishing HttpLogging --- eng/ProjectReferences.props | 1 + eng/SharedFramework.Local.props | 1 + .../HttpLogging.Sample.csproj | 2 + .../samples/HttpLogging.Sample/Startup.cs | 10 +- .../appsettings.Development.json | 2 +- .../HttpLogging.Sample/appsettings.json | 2 +- .../HttpLogging/src/HttpLoggingFields.cs | 96 ++ .../HttpLogging/src/HttpLoggingMiddleware.cs | 215 +++-- .../HttpLogging/src/HttpLoggingOptions.cs | 84 +- .../HttpLogging/src/HttpRequestLog.cs | 24 +- .../HttpLogging/src/HttpResponseLog.cs | 24 +- .../HttpLogging/src/PublicAPI.Unshipped.txt | 38 +- .../src/ResponseBufferingStream.cs | 229 +++-- .../test/HttpLoggingDefaultBuilderTests.cs | 12 + .../test/HttpLoggingMiddlewareTests.cs | 817 ++++++++++++++++++ .../HttpLogging/test/HttpLoggingTests.cs | 407 --------- ...rosoft.AspNetCore.HttpLogging.Tests.csproj | 2 +- src/Middleware/Middleware.slnf | 3 + 18 files changed, 1427 insertions(+), 542 deletions(-) create mode 100644 src/Middleware/HttpLogging/src/HttpLoggingFields.cs create mode 100644 src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs create mode 100644 src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs delete mode 100644 src/Middleware/HttpLogging/test/HttpLoggingTests.cs diff --git a/eng/ProjectReferences.props b/eng/ProjectReferences.props index a0a18b9d018a..d42d0b35990a 100644 --- a/eng/ProjectReferences.props +++ b/eng/ProjectReferences.props @@ -83,6 +83,7 @@ + diff --git a/eng/SharedFramework.Local.props b/eng/SharedFramework.Local.props index 49d28f78f0f2..e29a6f2a2d30 100644 --- a/eng/SharedFramework.Local.props +++ b/eng/SharedFramework.Local.props @@ -69,6 +69,7 @@ + diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj index 971dcbc6f457..21b5a022f5a1 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/HttpLogging.Sample.csproj @@ -6,6 +6,8 @@ + + diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs index a251380910c0..e23b8f6be1ee 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.HttpLogging; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using System; @@ -16,17 +17,22 @@ public class Startup // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { + services.AddHttpLogging(logging => + { + logging.LoggingFields = HttpLoggingFields.All; + }); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { app.UseRouting(); - + app.UseHttpLogging(); app.UseEndpoints(endpoints => { - endpoints.MapGet("/", async context => + endpoints.Map("/", async context => { + context.Response.ContentType = "text/plain"; await context.Response.WriteAsync("Hello World!"); }); }); diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json index 8983e0fc1c5e..63ed6e14613c 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.Development.json @@ -2,7 +2,7 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft": "Warning", + "Microsoft": "Information", "Microsoft.Hosting.Lifetime": "Information" } } diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json index d9d9a9bff6fd..214d63f9b192 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/appsettings.json @@ -2,7 +2,7 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft": "Warning", + "Microsoft": "Information", "Microsoft.Hosting.Lifetime": "Information" } }, diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs new file mode 100644 index 000000000000..b9914ec8aaea --- /dev/null +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -0,0 +1,96 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.AspNetCore.HttpLogging +{ + /// + /// Logging enum to enable different request and response logging fields. + /// + public enum HttpLoggingFields + { + /// + /// None + /// + None = 0, + + /// + /// Request Path + /// + Path = 1, + + /// + /// Request Query String + /// + Query = 2, + + /// + /// Request Protocol + /// + Protocol = 4, + + /// + /// Request Method + /// + Method = 8, + + /// + /// Request Scheme + /// + Scheme = 0x10, + + /// + /// Response Status Code + /// + StatusCode = 0x20, + + /// + /// Request Headers + /// + RequestHeaders = 0x40, + + /// + /// Response Headers + /// + ResponseHeaders = 0x80, + + /// + /// Request Body + /// + RequestBody = 0x100, + + /// + /// Response Body + /// + ResponseBody = 0x200, + + /// + /// Combination of request properties, including Path, Query, Protocol, Method, and Scheme + /// + RequestProperties = Path | Query | Protocol | Method | Scheme, + + /// + /// Combination of Request Properties and Request Headers + /// + RequestPropertiesAndHeaders = RequestProperties | RequestHeaders, + + /// + /// Combination of Response Properties and Response Headers + /// + ResponsePropertiesAndHeaders = StatusCode | ResponseHeaders, + + /// + /// Entire Request + /// + Request = RequestPropertiesAndHeaders | RequestBody, + + /// + /// Entire Response + /// + Response = StatusCode | ResponseHeaders | ResponseBody, + + /// + /// Entire Request and Response + /// + All = Request | Response + } +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index d7231c2875bc..9be42c811c88 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Text; @@ -9,9 +10,9 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.Net.Http.Headers; namespace Microsoft.AspNetCore.HttpLogging { @@ -21,8 +22,6 @@ namespace Microsoft.AspNetCore.HttpLogging public class HttpLoggingMiddleware { private readonly RequestDelegate _next; - - private readonly IConfiguration _config; private readonly ILogger _logger; private HttpLoggingOptions _options; private const int PipeThreshold = 32 * 1024; @@ -32,19 +31,22 @@ public class HttpLoggingMiddleware /// /// /// - /// /// - public HttpLoggingMiddleware(RequestDelegate next, IOptions options, IConfiguration config, ILoggerFactory loggerFactory) + public HttpLoggingMiddleware(RequestDelegate next, IOptions options, ILoggerFactory loggerFactory) { _next = next ?? throw new ArgumentNullException(nameof(next)); - _config = config ?? throw new ArgumentNullException(nameof(config)); if (options == null) { throw new ArgumentNullException(nameof(options)); } + if (loggerFactory == null) + { + throw new ArgumentNullException(nameof(loggerFactory)); + } + _options = options.Value; _logger = loggerFactory.CreateLogger(); } @@ -56,36 +58,53 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptions /// public async Task Invoke(HttpContext context) { - if (!_options.DisableRequestLogging) + if ((HttpLoggingFields.Request & _options.LoggingFields) != HttpLoggingFields.None) { var list = new List>(); var request = context.Request; - list.Add(new KeyValuePair(nameof(request.Protocol), request.Protocol)); - list.Add(new KeyValuePair(nameof(request.Method), request.Method)); - list.Add(new KeyValuePair(nameof(request.ContentType), request.ContentType)); - list.Add(new KeyValuePair(nameof(request.ContentLength), request.ContentLength)); - list.Add(new KeyValuePair(nameof(request.Scheme), request.Scheme)); - list.Add(new KeyValuePair(nameof(request.Host), request.Host.Value)); - list.Add(new KeyValuePair(nameof(request.PathBase), request.PathBase.Value)); - list.Add(new KeyValuePair(nameof(request.Path), request.Path.Value)); - list.Add(new KeyValuePair(nameof(request.QueryString), request.QueryString.Value)); - // Would hope for this to be nested somehow? Scope? - foreach (var header in FilterHeaders(request.Headers)) + if (_options.LoggingFields.HasFlag(HttpLoggingFields.Protocol)) { - list.Add(header); + list.Add(new KeyValuePair(nameof(request.Protocol), request.Protocol)); } - // reading the request body always seems expensive. - // TODO do we want string here? Other middleware writes to utf8jsonwriter directly. - var body = await ReadRequestBody(request, context.RequestAborted); + if (_options.LoggingFields.HasFlag(HttpLoggingFields.Method)) + { + list.Add(new KeyValuePair(nameof(request.Method), request.Method)); + } - list.Add(new KeyValuePair(nameof(request.Body), body)); + if (_options.LoggingFields.HasFlag(HttpLoggingFields.Scheme)) + { + list.Add(new KeyValuePair(nameof(request.Scheme), request.Scheme)); + } - // TODO add and remove things from log. + if (_options.LoggingFields.HasFlag(HttpLoggingFields.Path)) + { + list.Add(new KeyValuePair(nameof(request.PathBase), request.PathBase.Value)); + list.Add(new KeyValuePair(nameof(request.Path), request.Path.Value)); + } + + if (_options.LoggingFields.HasFlag(HttpLoggingFields.Query)) + { + list.Add(new KeyValuePair(nameof(request.QueryString), request.QueryString.Value)); + } + + if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders)) + { + FilterHeaders(list, request.Headers, _options.AllowedRequestHeaders); + } + if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) && IsSupportedMediaType(request.ContentType)) + { + var body = await ReadRequestBody(request, context.RequestAborted); + + list.Add(new KeyValuePair(nameof(request.Body), body)); + } + + // TODO add and remove things from log. var httpRequestLog = new HttpRequestLog(list); + _logger.Log(LogLevel.Information, eventId: LoggerEventIds.RequestLog, state: httpRequestLog, @@ -93,7 +112,7 @@ public async Task Invoke(HttpContext context) formatter: HttpRequestLog.Callback); } - if (_options.DisableResponseLogging) + if ((HttpLoggingFields.Response & _options.LoggingFields) == HttpLoggingFields.None) { // Short circuit and don't replace response body. await _next(context).ConfigureAwait(false); @@ -101,73 +120,113 @@ public async Task Invoke(HttpContext context) } var response = context.Response; - var originalBody = response.Body; - // TODO pool memory streams. - - var originalBodyFeature = context.Features.Get()!; - var bufferingStream = new ResponseBufferingStream(originalBodyFeature, memoryStream, _options.ResponseBodyLogLimit); - response.Body = bufferingStream; + ResponseBufferingStream? bufferingStream = null; + IHttpResponseBodyFeature? originalBodyFeature = null; + if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) + { + originalBodyFeature = context.Features.Get()!; + // TODO pool these. + bufferingStream = new ResponseBufferingStream(originalBodyFeature, _options.ResponseBodyLogLimit); + response.Body = bufferingStream; + } try { await _next(context).ConfigureAwait(false); var list = new List>(); - // TODO elapsed milliseconds? - list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); - list.Add(new KeyValuePair(nameof(response.ContentType), response.ContentType)); - list.Add(new KeyValuePair(nameof(response.ContentLength), response.ContentLength)); + if (_options.LoggingFields.HasFlag(HttpLoggingFields.StatusCode)) + { + list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); + } + + if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) + { + FilterHeaders(list, response.Headers, _options.AllowedResponseHeaders); + } - var httpRequestLog = new HttpResponseLog(list); - foreach (var header in FilterHeaders(response.Headers)) + if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody) && IsSupportedMediaType(response.ContentType)) { - list.Add(header); + var body = bufferingStream!.GetString(_options.BodyEncoding); + list.Add(new KeyValuePair(nameof(response.Body), body)); } + + var httpResponseLog = new HttpResponseLog(list); + + _logger.Log(LogLevel.Information, + eventId: LoggerEventIds.ResponseLog, + state: httpResponseLog, + exception: null, + formatter: HttpResponseLog.Callback); } finally { - context.Features.Set(originalBodyFeature); + if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) + { + if (bufferingStream != null) + { + bufferingStream.Dispose(); + } + + context.Features.Set(originalBodyFeature); + } + } + } + + private bool IsSupportedMediaType(string contentType) + { + var mediaTypeList = _options.SupportedMediaTypes; + if (mediaTypeList == null || mediaTypeList.Count == 0 || string.IsNullOrEmpty(contentType)) + { + return false; } + + var mediaType = new MediaTypeHeaderValue(contentType); + foreach (var type in mediaTypeList) + { + if (mediaType.IsSubsetOf(type)) + { + return true; + } + } + + return false; } - private IEnumerable> FilterHeaders(IHeaderDictionary headers) + private void FilterHeaders(List> keyValues, IHeaderDictionary headers, ISet allowedHeaders) { foreach (var (key, value) in headers) { - //if (_options.Filtering == HttpHeaderFiltering.OnlyListed && !_options.FilteringSet.Contains(key)) - //{ - // // Key is not among the "only listed" headers. - // continue; - //} - - //if (_options.Filtering == HttpHeaderFiltering.AllButListed && _options.FilteringSet.Contains(key)) - //{ - // // Key is among "all but listed" headers. - // continue; - //} - - //if (_options.Redact && _options.Redactors.TryGetValue(key, out var redactor)) - //{ - // yield return (key, redactor(value.ToString())); - // continue; - //} - - yield return new KeyValuePair(key, value.ToString()); + if (!allowedHeaders.Contains(key)) + { + // Key is not among the "only listed" headers. + keyValues.Add(new KeyValuePair(key, "X")); + continue; + } + keyValues.Add(new KeyValuePair(key, value.ToString())); } } private async Task ReadRequestBody(HttpRequest request, CancellationToken token) { - using var joinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(); - joinedTokenSource.CancelAfter(_options.ReadTimeout); + if (_options.BodyEncoding == null) + { + return "X"; + } + + using var joinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); + joinedTokenSource.CancelAfter(_options.RequestBodyTimeout); var limit = _options.RequestBodyLogLimit; + + // Use a pipe for smaller payload sizes if (limit <= PipeThreshold) { try { while (true) { + // TODO if someone uses the body after this, it will not have the rest of the data. var result = await request.BodyReader.ReadAsync(joinedTokenSource.Token); if (!result.IsCompleted && result.Buffer.Length <= limit) { @@ -176,7 +235,8 @@ private async Task ReadRequestBody(HttpRequest request, CancellationToke continue; } - var res = Encoding.UTF8.GetString(result.Buffer.Slice(0, result.Buffer.Length > limit ? limit : result.Buffer.Length)); + var res = _options.BodyEncoding.GetString(result.Buffer.Slice(0, result.Buffer.Length > limit ? limit : result.Buffer.Length)); + request.BodyReader.AdvanceTo(result.Buffer.Start, result.Buffer.End); return res; } @@ -186,8 +246,7 @@ private async Task ReadRequestBody(HttpRequest request, CancellationToke // Token source hides triggering token (https://github.com/dotnet/runtime/issues/22172) if (!token.IsCancellationRequested && joinedTokenSource.Token.IsCancellationRequested) { - // TODO should this be empty instead? - return "[Cancelled]"; + return "X"; } throw; @@ -195,18 +254,36 @@ private async Task ReadRequestBody(HttpRequest request, CancellationToke } else { - // TODO determine buffering limits here. request.EnableBuffering(); // Read here. + var buffer = ArrayPool.Shared.Rent(limit); - if (request.Body.CanSeek) + try { - _ = request.Body.Seek(0, SeekOrigin.Begin); + var count = 0; + while (true) + { + var read = await request.Body.ReadAsync(buffer, count, limit - count); + count += read; + if (read == 0 || count == limit) + { + break; + } + } + + return _options.BodyEncoding.GetString(new Span(buffer).Slice(0, count)); + } + finally + { + ArrayPool.Shared.Return(buffer); + // reseek back to start. + if (request.Body.CanSeek) + { + _ = request.Body.Seek(0, SeekOrigin.Begin); + } } } - - return ""; } } } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index ea0a7a6ab9f8..b7aae8d871e2 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -2,20 +2,90 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using Microsoft.AspNetCore.Http; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using Microsoft.Net.Http.Headers; namespace Microsoft.AspNetCore.HttpLogging { + /// + /// Options for the + /// public class HttpLoggingOptions { - // TODO make these reloadable. - public bool DisableRequestLogging { get; set; } = false; - public bool DisableResponseLogging { get; set; } = false; + /// + /// Fields to log for the Request and Response. Defaults to logging request and response properties and headers. + /// + public HttpLoggingFields LoggingFields { get; set; } = HttpLoggingFields.RequestPropertiesAndHeaders | HttpLoggingFields.ResponsePropertiesAndHeaders; - public int RequestBodyLogLimit { get; set; } = 32 * 1024; // 32KB - public TimeSpan ReadTimeout { get; set; } = TimeSpan.FromSeconds(1); + /// + /// Request header values that are allowed to be logged. + /// + /// + /// If a request header is not present in the , + /// the header name will be logged with a redacted value. + /// + public ISet AllowedRequestHeaders { get; } = new HashSet() + { + HeaderNames.Accept, + HeaderNames.AcceptEncoding, + HeaderNames.AcceptLanguage, + HeaderNames.Allow, + HeaderNames.Connection, + HeaderNames.ContentType, + HeaderNames.Host, + HeaderNames.UserAgent + }; - public int ResponseBodyLogLimit { get; set; } = 32 * 1024; // 32KB + /// + /// Response header values that are allowed to be logged. + /// + /// + /// If a response header is not present in the , + /// the header name will be logged with a redacted value. + /// + public ISet AllowedResponseHeaders { get; } = new HashSet() + { + HeaderNames.ContentType, + HeaderNames.Date, + HeaderNames.ETag, + HeaderNames.Server, + HeaderNames.TransferEncoding + }; + + /// + /// The encoding to log the request body and response body, if the content-type of the body + /// is one of the . + /// + public Encoding? BodyEncoding { get; set; } = Encoding.UTF8; + + /// + /// A list of supported media type values for request and response body logging. + /// + /// + /// If the request or response do not match the supported media type, the response body will not be logged. + /// + public List? SupportedMediaTypes { get; } = new List() + { + new MediaTypeHeaderValue("application/json"), + new MediaTypeHeaderValue("application/xml"), + new MediaTypeHeaderValue("text/*") + }; + /// + /// Maximum request body size to log (in bytes). Defaults to 32 KB. + /// + public int RequestBodyLogLimit { get; set; } = 32 * 1024; + + /// + /// Timeout for reading requset body to log. + /// + public TimeSpan RequestBodyTimeout { get; set; } = Debugger.IsAttached ? TimeSpan.FromDays(1) : TimeSpan.FromSeconds(1); + + /// + /// Maximum response body size to log (in bytes). Defaults to 32 KB. + /// + public int ResponseBodyLogLimit { get; set; } = 32 * 1024; // 32KB } } diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs index 03bf3a7dbc77..179e5fbe6f70 100644 --- a/src/Middleware/HttpLogging/src/HttpRequestLog.cs +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -4,6 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Text; namespace Microsoft.AspNetCore.HttpLogging { @@ -35,8 +36,27 @@ public override string ToString() { if (_cachedToString == null) { - // TODO new line separated list? - _cachedToString = ""; + var builder = new StringBuilder(); + var count = _keyValues.Count; + + for (var i = 0; i < count - 1; i++) + { + var kvp = _keyValues[i]; + builder.Append(kvp.Key); + builder.Append(": "); + builder.Append(kvp.Value); + builder.Append(Environment.NewLine); + } + + if (count > 0) + { + var kvp = _keyValues[count - 1]; + builder.Append(kvp.Key); + builder.Append(": "); + builder.Append(kvp.Value); + } + + _cachedToString = builder.ToString(); } return _cachedToString; diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs index aa50e1bb496f..a6684dfe5ac1 100644 --- a/src/Middleware/HttpLogging/src/HttpResponseLog.cs +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -4,6 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Text; namespace Microsoft.AspNetCore.HttpLogging { @@ -35,8 +36,27 @@ public override string ToString() { if (_cachedToString == null) { - // TODO new line separated list? - _cachedToString = ""; + var builder = new StringBuilder(); + var count = _keyValues.Count; + + for (var i = 0; i < count - 1; i++) + { + var kvp = _keyValues[i]; + builder.Append(kvp.Key); + builder.Append(": "); + builder.Append(kvp.Value); + builder.Append(Environment.NewLine); + } + + if (count > 0) + { + var kvp = _keyValues[count - 1]; + builder.Append(kvp.Key); + builder.Append(": "); + builder.Append(kvp.Value); + } + + _cachedToString = builder.ToString(); } return _cachedToString; diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index d0af8d8b4f5c..f2907e75032e 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -1,6 +1,42 @@ #nullable enable Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions +Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.All = 1023 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Method = 8 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.None = 0 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Path = 1 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Protocol = 4 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Query = 2 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request = 351 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestBody = 256 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestHeaders = 64 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties = 31 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPropertiesAndHeaders = 95 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response = 672 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseBody = 512 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders = 128 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders = 160 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Scheme = 16 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode = 32 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.Invoke(Microsoft.AspNetCore.Http.HttpContext! context) -> System.Threading.Tasks.Task! +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedRequestHeaders.get -> System.Collections.Generic.ISet! +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedResponseHeaders.get -> System.Collections.Generic.ISet! +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.BodyEncoding.get -> System.Text.Encoding? +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.BodyEncoding.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.HttpLoggingOptions() -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.get -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.get -> int +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyTimeout.get -> System.TimeSpan +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyTimeout.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.get -> int +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List? static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! -~Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.HttpLoggingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate! next, Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Configuration.IConfiguration! config, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void +static Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +~Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.HttpLoggingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate! next, Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 5e374ef86420..e095faa53810 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -4,9 +4,11 @@ using System; using System.Buffers; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Pipelines; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; @@ -14,7 +16,7 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature + internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBufferWriter { private readonly IHttpResponseBodyFeature _innerBodyFeature; private readonly Stream _innerStream; @@ -65,11 +67,6 @@ public PipeWriter Writer } } - public ReadOnlyMemory GetBuffer() - { - return _buffer.ToArray(); - } - public override int Read(byte[] buffer, int offset, int count) { throw new NotSupportedException(); @@ -97,12 +94,7 @@ public override Task FlushAsync(CancellationToken cancellationToken) public override void Write(byte[] buffer, int offset, int count) { - var remaining = _limit - (int)_buffer.Length; - if (remaining > 0) - { - _buffer.Write(buffer, offset, Math.Min(count, remaining)); - } - _innerStream.Write(buffer, offset, count); + Write(new ReadOnlySpan(buffer, offset, count)); } public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) @@ -152,12 +144,42 @@ public override void EndWrite(IAsyncResult asyncResult) task.GetAwaiter().GetResult(); } +#if NETCOREAPP + public override void Write(ReadOnlySpan span) + { + var remaining = _limit - _bytesWritten; + var innerCount = Math.Min(remaining, span.Length); + + if (_currentSegment != null && span.Slice(0, innerCount).TryCopyTo(_currentSegment.AsSpan(_position))) + { + _position += innerCount; + _bytesWritten += innerCount; + } + else + { + BuffersExtensions.Write(this, span.Slice(0, innerCount)); + } + + _innerStream.Write(span); + } +#endif + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - var remaining = _limit - (int)_buffer.Length; - if (remaining > 0) + var remaining = _limit - _bytesWritten; + var innerCount = Math.Min(remaining, count); + + var position = _position; + if (_currentSegment != null && position < _currentSegment.Length - innerCount) + { + Buffer.BlockCopy(buffer, offset, _currentSegment, position, innerCount); + + _position = position + innerCount; + _bytesWritten += innerCount; + } + else { - await _buffer.WriteAsync(buffer, offset, Math.Min(count, remaining)); + BuffersExtensions.Write(this, buffer.AsSpan(offset, innerCount)); } await _innerStream.WriteAsync(buffer, offset, count, cancellationToken); @@ -183,6 +205,29 @@ public async Task CompleteAsync() await _innerBodyFeature.CompleteAsync(); } + // IBufferWriter + public void Reset() + { + if (_completedSegments != null) + { + for (var i = 0; i < _completedSegments.Count; i++) + { + _completedSegments[i].Return(); + } + + _completedSegments.Clear(); + } + + if (_currentSegment != null) + { + ArrayPool.Shared.Return(_currentSegment); + _currentSegment = null; + } + + _bytesWritten = 0; + _position = 0; + } + public void Advance(int count) { _bytesWritten += count; @@ -202,50 +247,31 @@ public Span GetSpan(int sizeHint = 0) return _currentSegment.AsSpan(_position, _currentSegment.Length - _position); } - public override void WriteByte(byte value) + + public void CopyTo(IBufferWriter destination) { - if (_currentSegment != null && (uint)_position < (uint)_currentSegment.Length) + if (_completedSegments != null) { - _currentSegment[_position] = value; - } - else - { - AddSegment(); - _currentSegment[0] = value; + // Copy completed segments + var count = _completedSegments.Count; + for (var i = 0; i < count; i++) + { + destination.Write(_completedSegments[i].Span); + } } - _position++; - _bytesWritten++; + destination.Write(_currentSegment.AsSpan(0, _position)); } - public override void Write(byte[] buffer, int offset, int count) + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) { - var position = _position; - if (_currentSegment != null && position < _currentSegment.Length - count) - { - Buffer.BlockCopy(buffer, offset, _currentSegment, position, count); - - _position = position + count; - _bytesWritten += count; - } - else + if (_completedSegments == null && _currentSegment is not null) { - BuffersExtensions.Write(this, buffer.AsSpan(offset, count)); + // There is only one segment so write without awaiting. + return destination.WriteAsync(_currentSegment, 0, _position); } - } -#if NETCOREAPP - public override void Write(ReadOnlySpan span) - { - if (_currentSegment != null && span.TryCopyTo(_currentSegment.AsSpan(_position))) - { - _position += span.Length; - _bytesWritten += span.Length; - } - else - { - BuffersExtensions.Write(this, span); - } + return CopyToSlowAsync(destination); } [MemberNotNull(nameof(_currentSegment))] @@ -290,6 +316,111 @@ private void AddSegment(int sizeHint = 0) _position = 0; } + private async Task CopyToSlowAsync(Stream destination) + { + if (_completedSegments != null) + { + // Copy full segments + var count = _completedSegments.Count; + for (var i = 0; i < count; i++) + { + var segment = _completedSegments[i]; + await destination.WriteAsync(segment.Buffer, 0, segment.Length); + } + } + + if (_currentSegment is not null) + { + await destination.WriteAsync(_currentSegment, 0, _position); + } + } + + public override void WriteByte(byte value) + { + if (_currentSegment != null && (uint)_position < (uint)_currentSegment.Length) + { + _currentSegment[_position] = value; + } + else + { + AddSegment(); + _currentSegment[0] = value; + } + + _position++; + _bytesWritten++; + } + + // TODO inefficient, don't want to allocate array size of string, + // but issues with decoder APIs returning character count larger than + // required. + public string GetString(Encoding? encoding) + { + if (_currentSegment == null || encoding == null) + { + return ""; + } + + var result = new byte[_bytesWritten]; + + var totalWritten = 0; + + if (_completedSegments != null) + { + // Copy full segments + var count = _completedSegments.Count; + for (var i = 0; i < count; i++) + { + var segment = _completedSegments[i]; + segment.Span.CopyTo(result.AsSpan(totalWritten)); + totalWritten += segment.Span.Length; + } + } + + // Copy current incomplete segment + _currentSegment.AsSpan(0, _position).CopyTo(result.AsSpan(totalWritten)); + + // If encoding is nullable + return encoding.GetString(result); + } + + public void CopyTo(Span span) + { + Debug.Assert(span.Length >= _bytesWritten); + + if (_currentSegment == null) + { + return; + } + + var totalWritten = 0; + + if (_completedSegments != null) + { + // Copy full segments + var count = _completedSegments.Count; + for (var i = 0; i < count; i++) + { + var segment = _completedSegments[i]; + segment.Span.CopyTo(span.Slice(totalWritten)); + totalWritten += segment.Span.Length; + } + } + + // Copy current incomplete segment + _currentSegment.AsSpan(0, _position).CopyTo(span.Slice(totalWritten)); + + Debug.Assert(_bytesWritten == totalWritten + _position); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + Reset(); + } + } + /// /// Holds a byte[] from the pool and a size value. Basically a Memory but guaranteed to be backed by an ArrayPool byte[], so that we know we can return it. /// diff --git a/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs new file mode 100644 index 000000000000..b7f5f2c5a83f --- /dev/null +++ b/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.HttpLogging.Tests +{ + class HttpLoggingDefaultBuilderTests + { + } +} diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs new file mode 100644 index 000000000000..fb9880223288 --- /dev/null +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -0,0 +1,817 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.HttpLogging; +using Microsoft.AspNetCore.Testing; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.AspNetCore.HttpsPolicy.Tests +{ + public class HttpLoggingMiddlewareTests : LoggedTest + { + public static TheoryData BodyData + { + get + { + var variations = new TheoryData(); + variations.Add("Hello World"); + variations.Add(new string('a', 4097)); + variations.Add(new string('b', 10000)); + variations.Add(new string('あ', 10000)); + return variations; + } + } + + [Fact] + public void Ctor_ThrowsExceptionsWhenNullArgs() + { + Assert.Throws(() => new HttpLoggingMiddleware(null, CreateOptionsAccessor(), LoggerFactory)); + + Assert.Throws(() => new HttpLoggingMiddleware(c => + { + return Task.CompletedTask; + }, + null, + LoggerFactory)); + + Assert.Throws(() => new HttpLoggingMiddleware(c => + { + return Task.CompletedTask; + }, + CreateOptionsAccessor(), + null)); + } + + [Fact] + public async Task NoopWhenLoggingDisabled() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.None; + + var middleware = new HttpLoggingMiddleware( + c => + { + c.Response.StatusCode = 200; + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Protocol = "HTTP/1.0"; + httpContext.Request.Method = "GET"; + httpContext.Request.Scheme = "http"; + httpContext.Request.Path = new PathString("/foo"); + httpContext.Request.PathBase = new PathString("/foo"); + httpContext.Request.QueryString = new QueryString("?foo"); + httpContext.Request.Headers["Connection"] = "keep-alive"; + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes("test")); + + await middleware.Invoke(httpContext); + + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Protocol: HTTP/1.0")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Method: GET")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Scheme: http")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Path: /foo")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("PathBase: /foo")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("QueryString: ?foo")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + } + + [Fact] + public async Task DefaultRequestInfoOnlyHeadersAndRequestInfo() + { + var middleware = new HttpLoggingMiddleware( + c => + { + return Task.CompletedTask; + }, + CreateOptionsAccessor(), + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Protocol = "HTTP/1.0"; + httpContext.Request.Method = "GET"; + httpContext.Request.Scheme = "http"; + httpContext.Request.Path = new PathString("/foo"); + httpContext.Request.PathBase = new PathString("/foo"); + httpContext.Request.QueryString = new QueryString("?foo"); + httpContext.Request.Headers["Connection"] = "keep-alive"; + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes("test")); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Protocol: HTTP/1.0")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Method: GET")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Scheme: http")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Path: /foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("PathBase: /foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("QueryString: ?foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task RequestLogsAllRequestInfo() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.Request; + var middleware = new HttpLoggingMiddleware( + c => + { + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Protocol = "HTTP/1.0"; + httpContext.Request.Method = "GET"; + httpContext.Request.Scheme = "http"; + httpContext.Request.Path = new PathString("/foo"); + httpContext.Request.PathBase = new PathString("/foo"); + httpContext.Request.QueryString = new QueryString("?foo"); + httpContext.Request.Headers["Connection"] = "keep-alive"; + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes("test")); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Protocol: HTTP/1.0")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Method: GET")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Scheme: http")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Path: /foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("PathBase: /foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("QueryString: ?foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task RequestPropertiesLogs() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestProperties; + var middleware = new HttpLoggingMiddleware( + c => + { + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Protocol = "HTTP/1.0"; + httpContext.Request.Method = "GET"; + httpContext.Request.Scheme = "http"; + httpContext.Request.Path = new PathString("/foo"); + httpContext.Request.PathBase = new PathString("/foo"); + httpContext.Request.QueryString = new QueryString("?foo"); + httpContext.Request.Headers["Connection"] = "keep-alive"; + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes("test")); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Protocol: HTTP/1.0")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Method: GET")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Scheme: http")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Path: /foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("PathBase: /foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("QueryString: ?foo")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task RequestHeadersLogs() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestHeaders; + var middleware = new HttpLoggingMiddleware( + c => + { + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Protocol = "HTTP/1.0"; + httpContext.Request.Method = "GET"; + httpContext.Request.Scheme = "http"; + httpContext.Request.Path = new PathString("/foo"); + httpContext.Request.PathBase = new PathString("/foo"); + httpContext.Request.QueryString = new QueryString("?foo"); + httpContext.Request.Headers["Connection"] = "keep-alive"; + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes("test")); + + await middleware.Invoke(httpContext); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Protocol: HTTP/1.0")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Method: GET")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Scheme: http")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Path: /foo")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("PathBase: /foo")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("QueryString: ?foo")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task UnknownRequestHeadersRedacted() + { + var middleware = new HttpLoggingMiddleware( + c => + { + return Task.CompletedTask; + }, + CreateOptionsAccessor(), + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + httpContext.Request.Headers["foo"] = "bar"; + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: X")); + } + + [Fact] + public async Task CanConfigureRequestAllowList() + { + var options = CreateOptionsAccessor(); + options.Value.AllowedRequestHeaders.Clear(); + options.Value.AllowedRequestHeaders.Add("foo"); + var middleware = new HttpLoggingMiddleware( + c => + { + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + // Header on the default allow list. + httpContext.Request.Headers["Connection"] = "keep-alive"; + + httpContext.Request.Headers["foo"] = "bar"; + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: bar")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: X")); + } + + [Theory] + [MemberData(nameof(BodyData))] + public async Task RequestBodyReadingWorks(string expected) + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(expected)); + + await middleware.Invoke(httpContext); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Fact] + public async Task RequestBodyReadingLimitWorks() + { + var input = string.Concat(new string('a', 30000), new string('b', 3000)); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); + + await middleware.Invoke(httpContext); + var expected = input.Substring(0, options.Value.ResponseBodyLogLimit); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Fact] + public async Task RequestBodyReadingLimitGreaterThanPipeWorks() + { + var input = string.Concat(new string('a', 60000), new string('b', 3000)); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var count = 0; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + count += res; + } + + Assert.Equal(63000, count); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); + + await middleware.Invoke(httpContext); + var expected = input.Substring(0, options.Value.ResponseBodyLogLimit); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Theory] + [InlineData("text/plain")] + [InlineData("text/html")] + [InlineData("application/json")] + [InlineData("application/xml")] + [InlineData("application/entity+json")] + [InlineData("application/entity+xml")] + public async Task VerifyDefaultMediaTypeHeaders(string contentType) + { + // media headers that should work. + var expected = new string('a', 1000); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = contentType; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(expected)); + + await middleware.Invoke(httpContext); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Theory] + [InlineData("application/invalid")] + [InlineData("multipart/form-data")] + public async Task RejectedContentTypes(string contentType) + { + // media headers that should work. + var expected = new string('a', 1000); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var count = 0; + + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + count += res; + } + + Assert.Equal(1000, count); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = contentType; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(expected)); + + await middleware.Invoke(httpContext); + + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Fact] + public async Task DifferentEncodingsWork() + { + var encoding = Encoding.Unicode; + var expected = new string('a', 1000); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.Value.BodyEncoding = encoding; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var count = 0; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + count += res; + } + + Assert.Equal(1000, count); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(encoding.GetBytes(expected)); + + await middleware.Invoke(httpContext); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Fact] + public async Task DefaultBodyTimeoutTruncates() + { + // media headers that should work. + var expected = new string('a', 1000); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var count = 0; + + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + count += res; + } + + Assert.Equal(1000, count); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new SlowStream(new MemoryStream(Encoding.ASCII.GetBytes("test")), TimeSpan.FromSeconds(2)); + + await Assert.ThrowsAsync(async () => await middleware.Invoke(httpContext)); + } + + [Fact] + public async Task CanSetTimeoutToDifferentValue() + { + // media headers that should work. + var expected = new string('a', 1000); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.Value.RequestBodyTimeout = TimeSpan.FromSeconds(30); + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var count = 0; + + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + count += res; + } + + Assert.Equal(1000, count); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + // Does not timeout. + httpContext.Request.Body = new SlowStream(new MemoryStream(Encoding.ASCII.GetBytes("test")), TimeSpan.FromSeconds(2)); + + await middleware.Invoke(httpContext); + } + + [Fact] + public async Task DefaultResponseInfoOnlyHeadersAndRequestInfo() + { + var middleware = new HttpLoggingMiddleware( + async c => + { + c.Response.StatusCode = 200; + c.Response.Headers["Server"] = "Kestrel"; + c.Response.ContentType = "text/plain"; + await c.Response.WriteAsync("test"); + }, + CreateOptionsAccessor(), + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task ResponseInfoLogsAll() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.Response; + + var middleware = new HttpLoggingMiddleware( + async c => + { + c.Response.StatusCode = 200; + c.Response.Headers["Server"] = "Kestrel"; + c.Response.ContentType = "text/plain"; + await c.Response.WriteAsync("test"); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + + [Fact] + public async Task StatusCodeLogs() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.StatusCode; + + var middleware = new HttpLoggingMiddleware( + async c => + { + c.Response.StatusCode = 200; + c.Response.Headers["Server"] = "Kestrel"; + c.Response.ContentType = "text/plain"; + await c.Response.WriteAsync("test"); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task ResponseHeadersLogs() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.ResponseHeaders; + + var middleware = new HttpLoggingMiddleware( + async c => + { + c.Response.StatusCode = 200; + c.Response.Headers["Server"] = "Kestrel"; + c.Response.ContentType = "text/plain"; + await c.Response.WriteAsync("test"); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task ResponseHeadersRedacted() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.ResponseHeaders; + + var middleware = new HttpLoggingMiddleware( + c => + { + c.Response.Headers["Test"] = "Kestrel"; + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Test: X")); + } + + [Fact] + public async Task AllowedResponseHeadersModify() + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.ResponseHeaders; + options.Value.AllowedResponseHeaders.Clear(); + options.Value.AllowedResponseHeaders.Add("Test"); + + var middleware = new HttpLoggingMiddleware( + c => + { + c.Response.Headers["Test"] = "Kestrel"; + c.Response.Headers["Server"] = "Kestrel"; + return Task.CompletedTask; + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Test: Kestrel")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: X")); + } + + [Theory] + [MemberData(nameof(BodyData))] + public async Task ResponseBodyWritingWorks(string expected) + { + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.ResponseBody; + var middleware = new HttpLoggingMiddleware( + c => + { + c.Response.ContentType = "text/plain"; + return c.Response.WriteAsync(expected); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + [Fact] + public async Task ResponseBodyWritingLimitWorks() + { + var input = string.Concat(new string('a', 30000), new string('b', 3000)); + var options = CreateOptionsAccessor(); + options.Value.LoggingFields = HttpLoggingFields.ResponseBody; + var middleware = new HttpLoggingMiddleware( + c => + { + c.Response.ContentType = "text/plain"; + return c.Response.WriteAsync(input); + }, + options, + LoggerFactory); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + + var expected = input.Substring(0, options.Value.ResponseBodyLogLimit); + Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + } + + private IOptions CreateOptionsAccessor() + { + var options = new HttpLoggingOptions(); + var optionsAccessor = Mock.Of>(o => o.Value == options); + return optionsAccessor; + } + + private class SlowStream : Stream + { + private readonly Stream _inner; + private readonly TimeSpan _artificialDelay; + + public SlowStream(Stream inner, TimeSpan artificialDelay) + { + _inner = inner; + _artificialDelay = artificialDelay; + } + + public override void Flush() + { + _inner.Flush(); + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken token) + { + await Task.Delay(_artificialDelay); + return await _inner.ReadAsync(buffer, offset, count, token); + } + + public override void SetLength(long value) + { + _inner.SetLength(value); + } + + public override void Write(byte[] buffer, int offset, int count) + { + _inner.Write(buffer, offset, count); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => _inner.CanWrite; + public override long Length => _inner.Length; + public override long Position + { + get => _inner.Position; + set => _inner.Position = value; + } + } + } +} diff --git a/src/Middleware/HttpLogging/test/HttpLoggingTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingTests.cs deleted file mode 100644 index e97e3226fb62..000000000000 --- a/src/Middleware/HttpLogging/test/HttpLoggingTests.cs +++ /dev/null @@ -1,407 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.TestHost; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Testing; -using Microsoft.Net.Http.Headers; -using Xunit; - -namespace Microsoft.AspNetCore.HttpsPolicy.Tests -{ - public class HttpLoggingTests - { - [Fact] - public async Task SetOptionsWithDefault_SetsMaxAgeToCorrectValue() - { - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri("https://example.com:5050"); - - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Equal("max-age=2592000", response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); - } - - [Theory] - [InlineData(0, false, false, "max-age=0")] - [InlineData(-1, false, false, "max-age=-1")] - [InlineData(0, true, false, "max-age=0; includeSubDomains")] - [InlineData(50000, false, true, "max-age=50000; preload")] - [InlineData(0, true, true, "max-age=0; includeSubDomains; preload")] - [InlineData(50000, true, true, "max-age=50000; includeSubDomains; preload")] - public async Task SetOptionsThroughConfigure_SetsHeaderCorrectly(int maxAge, bool includeSubDomains, bool preload, string expected) - { - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.Configure(options => - { - options.Preload = preload; - options.IncludeSubDomains = includeSubDomains; - options.MaxAge = TimeSpan.FromSeconds(maxAge); - }); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri("https://example.com:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Equal(expected, response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); - } - - [Theory] - [InlineData(0, false, false, "max-age=0")] - [InlineData(-1, false, false, "max-age=-1")] - [InlineData(0, true, false, "max-age=0; includeSubDomains")] - [InlineData(50000, false, true, "max-age=50000; preload")] - [InlineData(0, true, true, "max-age=0; includeSubDomains; preload")] - [InlineData(50000, true, true, "max-age=50000; includeSubDomains; preload")] - public async Task SetOptionsThroughHelper_SetsHeaderCorrectly(int maxAge, bool includeSubDomains, bool preload, string expected) - { - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddHsts(options => - { - options.Preload = preload; - options.IncludeSubDomains = includeSubDomains; - options.MaxAge = TimeSpan.FromSeconds(maxAge); - }); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri("https://example.com:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Equal(expected, response.Headers.GetValues(HeaderNames.StrictTransportSecurity).FirstOrDefault()); - } - - [Theory] - [InlineData("localhost")] - [InlineData("Localhost")] - [InlineData("LOCALHOST")] - [InlineData("127.0.0.1")] - [InlineData("[::1]")] - public async Task DefaultExcludesCommonLocalhostDomains_DoesNotSetHstsHeader(string hostUrl) - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri($"https://{hostUrl}:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Empty(response.Headers); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal($"The host '{hostUrl}' is excluded. Skipping HSTS header.", message.State.ToString(), ignoreCase: true); - } - - [Theory] - [InlineData("localhost")] - [InlineData("127.0.0.1")] - [InlineData("[::1]")] - public async Task AllowLocalhostDomainsIfListIsReset_SetHstsHeader(string hostUrl) - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - - services.AddHsts(options => - { - options.ExcludedHosts.Clear(); - }); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri($"https://{hostUrl}:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Single(response.Headers); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Trace, message.LogLevel); - Assert.Equal("Adding HSTS header to response.", message.State.ToString()); - } - - [Theory] - [InlineData("example.com")] - [InlineData("Example.com")] - [InlineData("EXAMPLE.COM")] - public async Task AddExcludedDomains_DoesNotAddHstsHeader(string hostUrl) - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - - services.AddHsts(options => - { - options.ExcludedHosts.Add(hostUrl); - }); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri($"https://{hostUrl}:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Empty(response.Headers); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal($"The host '{hostUrl}' is excluded. Skipping HSTS header.", message.State.ToString(), ignoreCase: true); - } - - [Fact] - public async Task WhenRequestIsInsecure_DoesNotAddHstsHeader() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri("http://example.com:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Empty(response.Headers); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Debug, message.LogLevel); - Assert.Equal("The request is insecure. Skipping HSTS header.", message.State.ToString()); - } - - [Fact] - public async Task WhenRequestIsSecure_AddsHstsHeader() - { - var sink = new TestSink( - TestSink.EnableWithTypeName, - TestSink.EnableWithTypeName); - var loggerFactory = new TestLoggerFactory(sink, enabled: true); - - using var host = new HostBuilder() - .ConfigureWebHost(webHostBuilder => - { - webHostBuilder - .UseTestServer() - .ConfigureServices(services => - { - services.AddSingleton(loggerFactory); - }) - .Configure(app => - { - app.UseHsts(); - app.Run(context => - { - return context.Response.WriteAsync("Hello world"); - }); - }); - }).Build(); - - await host.StartAsync(); - var server = host.GetTestServer(); - var client = server.CreateClient(); - client.BaseAddress = new Uri("https://example.com:5050"); - var request = new HttpRequestMessage(HttpMethod.Get, ""); - - var response = await client.SendAsync(request); - - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.Contains(response.Headers, x => x.Key == HeaderNames.StrictTransportSecurity); - - var logMessages = sink.Writes.ToList(); - - Assert.Single(logMessages); - var message = logMessages.Single(); - Assert.Equal(LogLevel.Trace, message.LogLevel); - Assert.Equal("Adding HSTS header to response.", message.State.ToString()); - } - } -} diff --git a/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpLogging.Tests.csproj b/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpLogging.Tests.csproj index 7691bf5733f1..17c52be2ca50 100644 --- a/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpLogging.Tests.csproj +++ b/src/Middleware/HttpLogging/test/Microsoft.AspNetCore.HttpLogging.Tests.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Middleware/Middleware.slnf b/src/Middleware/Middleware.slnf index a670000801b9..f4f2843ee531 100644 --- a/src/Middleware/Middleware.slnf +++ b/src/Middleware/Middleware.slnf @@ -31,6 +31,9 @@ "src\\Middleware\\HostFiltering\\sample\\HostFilteringSample.csproj", "src\\Middleware\\HostFiltering\\src\\Microsoft.AspNetCore.HostFiltering.csproj", "src\\Middleware\\HostFiltering\\test\\Microsoft.AspNetCore.HostFiltering.Tests.csproj", + "src\\Middleware\\HttpLogging\\samples\\HttpLogging.Sample\\HttpLogging.Sample.csproj", + "src\\Middleware\\HttpLogging\\src\\Microsoft.AspNetCore.HttpLogging.csproj", + "src\\Middleware\\HttpLogging\\test\\Microsoft.AspNetCore.HttpLogging.Tests.csproj", "src\\Middleware\\HttpOverrides\\sample\\HttpOverridesSample.csproj", "src\\Middleware\\HttpOverrides\\src\\Microsoft.AspNetCore.HttpOverrides.csproj", "src\\Middleware\\HttpOverrides\\test\\Microsoft.AspNetCore.HttpOverrides.Tests.csproj", From e3c2bb9a778caeb8b7c1f3bf1b2d0381ea6751e0 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 14 Apr 2021 14:46:10 -0700 Subject: [PATCH 06/45] Namespace and nit --- src/Middleware/HttpLogging/src/HttpLoggingFields.cs | 1 + src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index b9914ec8aaea..85c3aab0d1f6 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -6,6 +6,7 @@ namespace Microsoft.AspNetCore.HttpLogging /// /// Logging enum to enable different request and response logging fields. /// + [Flags] public enum HttpLoggingFields { /// diff --git a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs index 0402bc550c7e..58bf16d3a343 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs @@ -6,7 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -namespace Microsoft.AspNetCore.Builder +namespace Microsoft.Extensions.DependencyInjection { /// /// Extension methods for the HttpLogging middleware. From a82072a85204daaf6ef7505f6e55641d2af1fa71 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 14 Apr 2021 15:05:46 -0700 Subject: [PATCH 07/45] System --- src/Middleware/HttpLogging/src/HttpLoggingFields.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index 85c3aab0d1f6..10764d8b0369 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; + namespace Microsoft.AspNetCore.HttpLogging { /// From 5b4d902bf01c9db2f595a9bb27a6200844e028c1 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 14 Apr 2021 19:32:39 -0700 Subject: [PATCH 08/45] Fix public API --- .../HttpLogging/src/PublicAPI.Unshipped.txt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index f2907e75032e..05db8d2c91cb 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -2,21 +2,21 @@ Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.All = 1023 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.All = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Method = 8 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.None = 0 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Path = 1 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Protocol = 4 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Query = 2 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request = 351 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPropertiesAndHeaders | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestBody -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestBody = 256 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestHeaders = 64 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties = 31 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPropertiesAndHeaders = 95 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response = 672 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Path | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Query | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Protocol | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Method | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Scheme -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPropertiesAndHeaders = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestHeaders -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseBody -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseBody = 512 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders = 128 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders = 160 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Scheme = 16 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode = 32 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware @@ -36,7 +36,10 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyTimeout.set -> vo Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List? +Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! static Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! static Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! ~Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.HttpLoggingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate! next, Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void From 704059b2743dc6ecb67c367fae2511bec536e78a Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 15 Apr 2021 12:37:59 -0700 Subject: [PATCH 09/45] Feedback --- .../HttpLogging/src/HttpLoggingFields.cs | 2 +- .../HttpLogging/src/HttpLoggingMiddleware.cs | 34 ++++++----- .../HttpLogging/src/HttpLoggingOptions.cs | 6 +- .../src/HttpLoggingServicesExtensions.cs | 16 ----- .../HttpLogging/src/HttpRequestLog.cs | 2 +- .../HttpLogging/src/HttpResponseLog.cs | 5 +- .../Microsoft.AspNetCore.HttpLogging.csproj | 1 + .../src/Properties/AssemblyInfo.cs | 3 + .../HttpLogging/src/PublicAPI.Unshipped.txt | 11 +--- .../src/ResponseBufferingStream.cs | 61 ++++--------------- .../test/HttpLoggingDefaultBuilderTests.cs | 2 +- 11 files changed, 46 insertions(+), 97 deletions(-) create mode 100644 src/Middleware/HttpLogging/src/Properties/AssemblyInfo.cs diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index 10764d8b0369..bb66ba1a0d1b 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.HttpLogging /// Logging enum to enable different request and response logging fields. /// [Flags] - public enum HttpLoggingFields + public enum HttpLoggingFields : long { /// /// None diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 9be42c811c88..f6a4f90ada70 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -4,6 +4,7 @@ using System; using System.Buffers; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading; @@ -19,12 +20,14 @@ namespace Microsoft.AspNetCore.HttpLogging /// /// Middleware that logs HTTP requests and HTTP responses. /// - public class HttpLoggingMiddleware + internal sealed class HttpLoggingMiddleware { private readonly RequestDelegate _next; private readonly ILogger _logger; private HttpLoggingOptions _options; private const int PipeThreshold = 32 * 1024; + private const int DefaultRequestFieldsMinusHeaders = 7; + private const int DefaultResponseFieldsMinusHeaders = 2; /// /// Initializes . @@ -60,34 +63,33 @@ public async Task Invoke(HttpContext context) { if ((HttpLoggingFields.Request & _options.LoggingFields) != HttpLoggingFields.None) { - var list = new List>(); - var request = context.Request; + var list = new List>(request.Headers.Count + DefaultRequestFieldsMinusHeaders); if (_options.LoggingFields.HasFlag(HttpLoggingFields.Protocol)) { - list.Add(new KeyValuePair(nameof(request.Protocol), request.Protocol)); + AddToList(list, nameof(request.Protocol), request.Protocol); } if (_options.LoggingFields.HasFlag(HttpLoggingFields.Method)) { - list.Add(new KeyValuePair(nameof(request.Method), request.Method)); + AddToList(list, nameof(request.Method), request.Method); } if (_options.LoggingFields.HasFlag(HttpLoggingFields.Scheme)) { - list.Add(new KeyValuePair(nameof(request.Scheme), request.Scheme)); + AddToList(list, nameof(request.Scheme), request.Scheme); } if (_options.LoggingFields.HasFlag(HttpLoggingFields.Path)) { - list.Add(new KeyValuePair(nameof(request.PathBase), request.PathBase.Value)); - list.Add(new KeyValuePair(nameof(request.Path), request.Path.Value)); + AddToList(list, nameof(request.PathBase), request.PathBase); + AddToList(list, nameof(request.Path), request.Path); } if (_options.LoggingFields.HasFlag(HttpLoggingFields.Query)) { - list.Add(new KeyValuePair(nameof(request.QueryString), request.QueryString.Value)); + AddToList(list, nameof(request.QueryString), request.QueryString.Value); } if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders)) @@ -134,7 +136,7 @@ public async Task Invoke(HttpContext context) try { await _next(context).ConfigureAwait(false); - var list = new List>(); + var list = new List>(response.Headers.Count + DefaultResponseFieldsMinusHeaders); if (_options.LoggingFields.HasFlag(HttpLoggingFields.StatusCode)) { @@ -164,16 +166,18 @@ public async Task Invoke(HttpContext context) { if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) { - if (bufferingStream != null) - { - bufferingStream.Dispose(); - } + bufferingStream?.Dispose(); context.Features.Set(originalBodyFeature); } } } + private static void AddToList(List> list, string key, string? value) + { + list.Add(new KeyValuePair(key, value)); + } + private bool IsSupportedMediaType(string contentType) { var mediaTypeList = _options.SupportedMediaTypes; @@ -266,6 +270,8 @@ private async Task ReadRequestBody(HttpRequest request, CancellationToke { var read = await request.Body.ReadAsync(buffer, count, limit - count); count += read; + + Debug.Assert(count <= limit); if (read == 0 || count == limit) { break; diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index b7aae8d871e2..100558f1222d 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -12,7 +12,7 @@ namespace Microsoft.AspNetCore.HttpLogging /// /// Options for the /// - public class HttpLoggingOptions + public sealed class HttpLoggingOptions { /// /// Fields to log for the Request and Response. Defaults to logging request and response properties and headers. @@ -26,7 +26,7 @@ public class HttpLoggingOptions /// If a request header is not present in the , /// the header name will be logged with a redacted value. /// - public ISet AllowedRequestHeaders { get; } = new HashSet() + public HashSet AllowedRequestHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.Accept, HeaderNames.AcceptEncoding, @@ -45,7 +45,7 @@ public class HttpLoggingOptions /// If a response header is not present in the , /// the header name will be logged with a redacted value. /// - public ISet AllowedResponseHeaders { get; } = new HashSet() + public HashSet AllowedResponseHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.ContentType, HeaderNames.Date, diff --git a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs index 58bf16d3a343..c057805eaf10 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs @@ -13,22 +13,6 @@ namespace Microsoft.Extensions.DependencyInjection /// public static class HttpLoggingServicesExtensions { - /// - /// Adds HTTP Logging services. - /// - /// The for adding services. - /// - public static IServiceCollection AddHttpLogging(this IServiceCollection services) - { - if (services == null) - { - throw new ArgumentNullException(nameof(services)); - } - - services.TryAddSingleton(); - return services; - } - /// /// Adds HTTP Logging services. /// diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs index 179e5fbe6f70..e6e09e0684cf 100644 --- a/src/Middleware/HttpLogging/src/HttpRequestLog.cs +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -8,7 +8,7 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal class HttpRequestLog : IReadOnlyList> + internal sealed class HttpRequestLog : IReadOnlyList> { private readonly List> _keyValues; private string? _cachedToString; diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs index a6684dfe5ac1..945afc2fb6af 100644 --- a/src/Middleware/HttpLogging/src/HttpResponseLog.cs +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -8,7 +8,7 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal class HttpResponseLog : IReadOnlyList> + internal sealed class HttpResponseLog : IReadOnlyList> { private readonly List> _keyValues; private string? _cachedToString; @@ -33,8 +33,7 @@ public HttpResponseLog(List> keyValues) } public override string ToString() - { - if (_cachedToString == null) + {if (_cachedToString == null) { var builder = new StringBuilder(); var count = _keyValues.Count; diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index fa0f5f6b7fc6..f91a5db5651e 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -18,5 +18,6 @@ + diff --git a/src/Middleware/HttpLogging/src/Properties/AssemblyInfo.cs b/src/Middleware/HttpLogging/src/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..62d2d373c4b8 --- /dev/null +++ b/src/Middleware/HttpLogging/src/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.AspNetCore.HttpLogging.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")] diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index 05db8d2c91cb..f35f6cce2272 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -1,6 +1,5 @@ #nullable enable Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions -Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.All = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Method = 8 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields @@ -19,11 +18,9 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders = 128 -> Micr Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Scheme = 16 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode = 32 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware -Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.Invoke(Microsoft.AspNetCore.Http.HttpContext! context) -> System.Threading.Tasks.Task! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedRequestHeaders.get -> System.Collections.Generic.ISet! -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedResponseHeaders.get -> System.Collections.Generic.ISet! +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedRequestHeaders.get -> System.Collections.Generic.HashSet! +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedResponseHeaders.get -> System.Collections.Generic.HashSet! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.BodyEncoding.get -> System.Text.Encoding? Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.BodyEncoding.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.HttpLoggingOptions() -> void @@ -38,8 +35,4 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List? Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! -static Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Microsoft.AspNetCore.Builder.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -~Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware.HttpLoggingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate! next, Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index e095faa53810..493adef0e0b6 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -25,10 +25,13 @@ internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBuff private readonly int _minimumSegmentSize; private int _bytesWritten; + private List? _completedSegments; private byte[]? _currentSegment; private int _position; + private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true); + internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit) { _innerBodyFeature = innerBodyFeature; @@ -60,7 +63,7 @@ public PipeWriter Writer { if (_pipeAdapter == null) { - _pipeAdapter = PipeWriter.Create(Stream, new StreamPipeWriterOptions(leaveOpen: true)); + _pipeAdapter = PipeWriter.Create(Stream, _pipeWriterOptions); } return _pipeAdapter; @@ -94,57 +97,19 @@ public override Task FlushAsync(CancellationToken cancellationToken) public override void Write(byte[] buffer, int offset, int count) { - Write(new ReadOnlySpan(buffer, offset, count)); + Write(buffer.AsSpan(offset, count)); } public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) { - var tcs = new TaskCompletionSource(state: state, TaskCreationOptions.RunContinuationsAsynchronously); - InternalWriteAsync(buffer, offset, count, callback, tcs); - return tcs.Task; - } - - private async void InternalWriteAsync(byte[] buffer, int offset, int count, AsyncCallback? callback, TaskCompletionSource tcs) - { - try - { - await WriteAsync(buffer, offset, count); - tcs.TrySetResult(); - } - catch (Exception ex) - { - tcs.TrySetException(ex); - } - - if (callback != null) - { - // Offload callbacks to avoid stack dives on sync completions. - var ignored = Task.Run(() => - { - try - { - callback(tcs.Task); - } - catch (Exception) - { - // Suppress exceptions on background threads. - } - }); - } + return TaskToApm.Begin(WriteAsync(buffer, offset, count), callback, state); } public override void EndWrite(IAsyncResult asyncResult) { - if (asyncResult == null) - { - throw new ArgumentNullException(nameof(asyncResult)); - } - - var task = (Task)asyncResult; - task.GetAwaiter().GetResult(); + TaskToApm.End(asyncResult); } -#if NETCOREAPP public override void Write(ReadOnlySpan span) { var remaining = _limit - _bytesWritten; @@ -162,7 +127,6 @@ public override void Write(ReadOnlySpan span) _innerStream.Write(span); } -#endif public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { @@ -300,10 +264,9 @@ private void AddSegment(int sizeHint = 0) if (_currentSegment != null) { // We're adding a segment to the list - if (_completedSegments == null) - { - _completedSegments = new List(); - } + // Initial size is limit / minimumSegmentSsize as that's the maximum size of this list. + // Round down because the last segment is in a field. + _completedSegments ??= new List(_limit / _minimumSegmentSize); // Position might be less than the segment length if there wasn't enough space to satisfy the sizeHint when // GetMemory was called. In that case we'll take the current segment and call it "completed", but need to @@ -361,7 +324,7 @@ public string GetString(Encoding? encoding) return ""; } - var result = new byte[_bytesWritten]; + var ros = new ReadOnlySequence(); var totalWritten = 0; @@ -381,7 +344,7 @@ public string GetString(Encoding? encoding) _currentSegment.AsSpan(0, _position).CopyTo(result.AsSpan(totalWritten)); // If encoding is nullable - return encoding.GetString(result); + return encoding.GetString(ros); } public void CopyTo(Span span) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs index b7f5f2c5a83f..f486c7ddc5ed 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; From b43466f6aa0b0ac5243108e1ece666a05da3d9c4 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 15 Apr 2021 13:57:32 -0700 Subject: [PATCH 10/45] Big perf wins for response body --- .../Microsoft.AspNetCore.HttpLogging.csproj | 1 + .../src/ResponseBufferingStream.cs | 273 +++++++----------- ...soft.AspNetCore.Server.Kestrel.Core.csproj | 1 + .../Buffers}/BufferSegment.cs | 0 .../Buffers}/BufferSegmentStack.cs | 0 5 files changed, 103 insertions(+), 172 deletions(-) rename src/{Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers => Shared/Buffers}/BufferSegment.cs (100%) rename src/{Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers => Shared/Buffers}/BufferSegmentStack.cs (100%) diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index f91a5db5651e..417a8db2fae5 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -19,5 +19,6 @@ + diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 493adef0e0b6..8b978430fdf6 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -8,6 +8,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Pipelines; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -18,17 +19,21 @@ namespace Microsoft.AspNetCore.HttpLogging { internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBufferWriter { + private const int MaxSegmentPoolSize = 256; // 1MB + private const int MinimumBufferSize = 4096; // 4K + private readonly IHttpResponseBodyFeature _innerBodyFeature; private readonly Stream _innerStream; private readonly int _limit; private PipeWriter? _pipeAdapter; - private readonly int _minimumSegmentSize; private int _bytesWritten; - private List? _completedSegments; - private byte[]? _currentSegment; - private int _position; + private readonly BufferSegmentStack _bufferSegmentPool; + private BufferSegment? _head; + private BufferSegment? _tail; + private Memory _tailMemory; // remainder of tail memory + private int _tailBytesBuffered; private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true); @@ -37,7 +42,7 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int _innerBodyFeature = innerBodyFeature; _innerStream = innerBodyFeature.Stream; _limit = limit; - _minimumSegmentSize = 4096; + _bufferSegmentPool = new BufferSegmentStack(limit / MinimumBufferSize); } public override bool CanRead => false; @@ -115,10 +120,11 @@ public override void Write(ReadOnlySpan span) var remaining = _limit - _bytesWritten; var innerCount = Math.Min(remaining, span.Length); - if (_currentSegment != null && span.Slice(0, innerCount).TryCopyTo(_currentSegment.AsSpan(_position))) + if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) { - _position += innerCount; + _tailBytesBuffered += innerCount; _bytesWritten += innerCount; + _tailMemory = _tailMemory.Slice(innerCount); } else { @@ -133,13 +139,13 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc var remaining = _limit - _bytesWritten; var innerCount = Math.Min(remaining, count); - var position = _position; - if (_currentSegment != null && position < _currentSegment.Length - innerCount) + if (_tailMemory.Length - innerCount > 0) { - Buffer.BlockCopy(buffer, offset, _currentSegment, position, innerCount); - - _position = position + innerCount; + //Buffer.BlockCopy(buffer, offset, , position, innerCount); + buffer.AsSpan(offset, count).CopyTo(_tailMemory.Span); + _tailBytesBuffered += innerCount; _bytesWritten += innerCount; + _tailMemory = _tailMemory.Slice(innerCount); } else { @@ -172,146 +178,128 @@ public async Task CompleteAsync() // IBufferWriter public void Reset() { - if (_completedSegments != null) - { - for (var i = 0; i < _completedSegments.Count; i++) - { - _completedSegments[i].Return(); - } - - _completedSegments.Clear(); - } - - if (_currentSegment != null) - { - ArrayPool.Shared.Return(_currentSegment); - _currentSegment = null; - } - _bytesWritten = 0; - _position = 0; + _tailBytesBuffered = 0; } - public void Advance(int count) + public void Advance(int bytes) { - _bytesWritten += count; - _position += count; + if ((uint)bytes > (uint)_tailMemory.Length) + { + ThrowArgumentOutOfRangeException(nameof(bytes)); + } + + _tailBytesBuffered += bytes; + _bytesWritten += bytes; + _tailMemory = _tailMemory.Slice(bytes); } public Memory GetMemory(int sizeHint = 0) { - EnsureCapacity(sizeHint); - - return _currentSegment.AsMemory(_position, _currentSegment.Length - _position); + AllocateMemoryUnsynchronized(sizeHint); + return _tailMemory; } public Span GetSpan(int sizeHint = 0) { - EnsureCapacity(sizeHint); - - return _currentSegment.AsSpan(_position, _currentSegment.Length - _position); + AllocateMemoryUnsynchronized(sizeHint); + return _tailMemory.Span; } - public void CopyTo(IBufferWriter destination) + private void AllocateMemoryUnsynchronized(int sizeHint) { - if (_completedSegments != null) + if (_head == null) + { + // We need to allocate memory to write since nobody has written before + BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + + // Set all the pointers + _head = _tail = newSegment; + _tailBytesBuffered = 0; + } + else { - // Copy completed segments - var count = _completedSegments.Count; - for (var i = 0; i < count; i++) + int bytesLeftInBuffer = _tailMemory.Length; + + if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < sizeHint) { - destination.Write(_completedSegments[i].Span); + Debug.Assert(_tail != null); + + if (_tailBytesBuffered > 0) + { + // Flush buffered data to the segment + _tail.End += _tailBytesBuffered; + _tailBytesBuffered = 0; + } + + BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + + _tail.SetNext(newSegment); + _tail = newSegment; } } - - destination.Write(_currentSegment.AsSpan(0, _position)); } - public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) { - if (_completedSegments == null && _currentSegment is not null) - { - // There is only one segment so write without awaiting. - return destination.WriteAsync(_currentSegment, 0, _position); - } + BufferSegment newSegment = CreateSegmentUnsynchronized(); + + // We can't use the recommended pool so use the ArrayPool + newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(Math.Max(MinimumBufferSize, sizeHint))); - return CopyToSlowAsync(destination); + _tailMemory = newSegment.AvailableMemory; + + return newSegment; } - [MemberNotNull(nameof(_currentSegment))] - private void EnsureCapacity(int sizeHint) + private BufferSegment CreateSegmentUnsynchronized() { - // This does the Right Thing. It only subtracts _position from the current segment length if it's non-null. - // If _currentSegment is null, it returns 0. - var remainingSize = _currentSegment?.Length - _position ?? 0; - - // If the sizeHint is 0, any capacity will do - // Otherwise, the buffer must have enough space for the entire size hint, or we need to add a segment. - if ((sizeHint == 0 && remainingSize > 0) || (sizeHint > 0 && remainingSize >= sizeHint)) + if (_bufferSegmentPool.TryPop(out var segment)) { - // We have capacity in the current segment -#pragma warning disable CS8774 // Member must have a non-null value when exiting. - return; -#pragma warning restore CS8774 // Member must have a non-null value when exiting. + return segment; } - AddSegment(sizeHint); + return new BufferSegment(); } - [MemberNotNull(nameof(_currentSegment))] - private void AddSegment(int sizeHint = 0) + private void ReturnSegmentUnsynchronized(BufferSegment segment) { - if (_currentSegment != null) + if (_bufferSegmentPool.Count < MaxSegmentPoolSize) { - // We're adding a segment to the list - // Initial size is limit / minimumSegmentSsize as that's the maximum size of this list. - // Round down because the last segment is in a field. - _completedSegments ??= new List(_limit / _minimumSegmentSize); - - // Position might be less than the segment length if there wasn't enough space to satisfy the sizeHint when - // GetMemory was called. In that case we'll take the current segment and call it "completed", but need to - // ignore any empty space in it. - _completedSegments.Add(new CompletedBuffer(_currentSegment, _position)); + _bufferSegmentPool.Push(segment); } - - // Get a new buffer using the minimum segment size, unless the size hint is larger than a single segment. - _currentSegment = ArrayPool.Shared.Rent(Math.Max(_minimumSegmentSize, sizeHint)); - _position = 0; } - private async Task CopyToSlowAsync(Stream destination) + private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) { - if (_completedSegments != null) - { - // Copy full segments - var count = _completedSegments.Count; - for (var i = 0; i < count; i++) - { - var segment = _completedSegments[i]; - await destination.WriteAsync(segment.Buffer, 0, segment.Length); - } - } + // First we need to handle case where hint is smaller than minimum segment size + sizeHint = Math.Max(MinimumBufferSize, sizeHint); + // After that adjust it to fit into pools max buffer size + var adjustedToMaximumSize = Math.Min(maxBufferSize, sizeHint); + return adjustedToMaximumSize; + } - if (_currentSegment is not null) - { - await destination.WriteAsync(_currentSegment, 0, _position); - } + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + throw new NotImplementedException(); } public override void WriteByte(byte value) { - if (_currentSegment != null && (uint)_position < (uint)_currentSegment.Length) + if (_bytesWritten == _limit) { - _currentSegment[_position] = value; + return; } - else + + if (_tailMemory.Length == 0) { - AddSegment(); - _currentSegment[0] = value; + AllocateMemoryUnsynchronized(MinimumBufferSize); } - _position++; + _tailMemory.Span[0] = value; + _tailBytesBuffered++; _bytesWritten++; + _tailMemory = _tailMemory.Slice(1); } // TODO inefficient, don't want to allocate array size of string, @@ -319,63 +307,21 @@ public override void WriteByte(byte value) // required. public string GetString(Encoding? encoding) { - if (_currentSegment == null || encoding == null) + if (_head == null || _tail == null || encoding == null) { return ""; } - var ros = new ReadOnlySequence(); + // Only place where we are actually using the buffered data. + // update tail here. + _tail.End = _tailBytesBuffered; - var totalWritten = 0; - - if (_completedSegments != null) - { - // Copy full segments - var count = _completedSegments.Count; - for (var i = 0; i < count; i++) - { - var segment = _completedSegments[i]; - segment.Span.CopyTo(result.AsSpan(totalWritten)); - totalWritten += segment.Span.Length; - } - } - - // Copy current incomplete segment - _currentSegment.AsSpan(0, _position).CopyTo(result.AsSpan(totalWritten)); + var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); // If encoding is nullable return encoding.GetString(ros); } - public void CopyTo(Span span) - { - Debug.Assert(span.Length >= _bytesWritten); - - if (_currentSegment == null) - { - return; - } - - var totalWritten = 0; - - if (_completedSegments != null) - { - // Copy full segments - var count = _completedSegments.Count; - for (var i = 0; i < count; i++) - { - var segment = _completedSegments[i]; - segment.Span.CopyTo(span.Slice(totalWritten)); - totalWritten += segment.Span.Length; - } - } - - // Copy current incomplete segment - _currentSegment.AsSpan(0, _position).CopyTo(span.Slice(totalWritten)); - - Debug.Assert(_bytesWritten == totalWritten + _position); - } - protected override void Dispose(bool disposing) { if (disposing) @@ -384,26 +330,9 @@ protected override void Dispose(bool disposing) } } - /// - /// Holds a byte[] from the pool and a size value. Basically a Memory but guaranteed to be backed by an ArrayPool byte[], so that we know we can return it. - /// - private readonly struct CompletedBuffer - { - public byte[] Buffer { get; } - public int Length { get; } - - public ReadOnlySpan Span => Buffer.AsSpan(0, Length); - - public CompletedBuffer(byte[] buffer, int length) - { - Buffer = buffer; - Length = length; - } - - public void Return() - { - ArrayPool.Shared.Return(Buffer); - } - } + // Copied from https://github.com/dotnet/corefx/blob/de3902bb56f1254ec1af4bf7d092fc2c048734cc/src/System.Memory/src/System/ThrowHelper.cs + private static void ThrowArgumentOutOfRangeException(string argumentName) { throw CreateArgumentOutOfRangeException(argumentName); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Exception CreateArgumentOutOfRangeException(string argumentName) { return new ArgumentOutOfRangeException(argumentName); } } } diff --git a/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj b/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj index ef7726f15f68..a9a6fe7986c3 100644 --- a/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj +++ b/src/Servers/Kestrel/Core/src/Microsoft.AspNetCore.Server.Kestrel.Core.csproj @@ -18,6 +18,7 @@ + diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/BufferSegment.cs b/src/Shared/Buffers/BufferSegment.cs similarity index 100% rename from src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/BufferSegment.cs rename to src/Shared/Buffers/BufferSegment.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/BufferSegmentStack.cs b/src/Shared/Buffers/BufferSegmentStack.cs similarity index 100% rename from src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/BufferSegmentStack.cs rename to src/Shared/Buffers/BufferSegmentStack.cs From 1efe8b3eda0d0f3a50da7e6d5630944935a6e976 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 15 Apr 2021 18:17:38 -0700 Subject: [PATCH 11/45] Adds request body side --- .../HttpLogging/src/CoreStrings.resx | 123 +++++++ .../HttpLogging/src/HttpLoggingMiddleware.cs | 140 +++----- .../HttpLogging/src/HttpLoggingOptions.cs | 10 +- .../HttpLogging/src/LoggerEventIds.cs | 2 + .../Microsoft.AspNetCore.HttpLogging.csproj | 1 + .../HttpLogging/src/PublicAPI.Unshipped.txt | 2 - .../HttpLogging/src/RequestBufferingStream.cs | 303 ++++++++++++++++++ .../src/ResponseBufferingStream.cs | 39 +-- .../test/HttpLoggingMiddlewareTests.cs | 76 +---- 9 files changed, 479 insertions(+), 217 deletions(-) create mode 100644 src/Middleware/HttpLogging/src/CoreStrings.resx create mode 100644 src/Middleware/HttpLogging/src/RequestBufferingStream.cs diff --git a/src/Middleware/HttpLogging/src/CoreStrings.resx b/src/Middleware/HttpLogging/src/CoreStrings.resx new file mode 100644 index 000000000000..2a560a73a722 --- /dev/null +++ b/src/Middleware/HttpLogging/src/CoreStrings.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + RequestBody: {0} + + \ No newline at end of file diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index f6a4f90ada70..b9108f688384 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -61,6 +61,8 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptions /// public async Task Invoke(HttpContext context) { + RequestBufferingStream? requestBufferingStream = null; + Stream? originalBody = null; if ((HttpLoggingFields.Request & _options.LoggingFields) != HttpLoggingFields.None) { var request = context.Request; @@ -97,12 +99,12 @@ public async Task Invoke(HttpContext context) FilterHeaders(list, request.Headers, _options.AllowedRequestHeaders); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) && IsSupportedMediaType(request.ContentType)) + if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody)) { - var body = await ReadRequestBody(request, context.RequestAborted); - - list.Add(new KeyValuePair(nameof(request.Body), body)); - } + originalBody = request.Body; + requestBufferingStream = new RequestBufferingStream(request.Body, _options.RequestBodyLogLimit, _logger, _options.BodyEncoding); + request.Body = requestBufferingStream; + } // TODO add and remove things from log. var httpRequestLog = new HttpRequestLog(list); @@ -114,27 +116,28 @@ public async Task Invoke(HttpContext context) formatter: HttpRequestLog.Callback); } - if ((HttpLoggingFields.Response & _options.LoggingFields) == HttpLoggingFields.None) - { - // Short circuit and don't replace response body. - await _next(context).ConfigureAwait(false); - return; - } - - var response = context.Response; - - ResponseBufferingStream? bufferingStream = null; + ResponseBufferingStream? responseBufferingStream = null; IHttpResponseBodyFeature? originalBodyFeature = null; - if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) - { - originalBodyFeature = context.Features.Get()!; - // TODO pool these. - bufferingStream = new ResponseBufferingStream(originalBodyFeature, _options.ResponseBodyLogLimit); - response.Body = bufferingStream; - } try { + if ((HttpLoggingFields.Response & _options.LoggingFields) == HttpLoggingFields.None) + { + // Short circuit and don't replace response body. + await _next(context).ConfigureAwait(false); + return; + } + + var response = context.Response; + + if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) + { + originalBodyFeature = context.Features.Get()!; + // TODO pool these. + responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, _options.ResponseBodyLogLimit, _logger); + response.Body = responseBufferingStream; + } + await _next(context).ConfigureAwait(false); var list = new List>(response.Headers.Count + DefaultResponseFieldsMinusHeaders); @@ -150,7 +153,7 @@ public async Task Invoke(HttpContext context) if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody) && IsSupportedMediaType(response.ContentType)) { - var body = bufferingStream!.GetString(_options.BodyEncoding); + var body = responseBufferingStream!.GetString(_options.BodyEncoding); list.Add(new KeyValuePair(nameof(response.Body), body)); } @@ -166,10 +169,16 @@ public async Task Invoke(HttpContext context) { if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) { - bufferingStream?.Dispose(); + responseBufferingStream!.Dispose(); context.Features.Set(originalBodyFeature); } + + if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody)) + { + requestBufferingStream!.Dispose(); + context.Request.Body = originalBody!; + } } } @@ -187,6 +196,9 @@ private bool IsSupportedMediaType(string contentType) } var mediaType = new MediaTypeHeaderValue(contentType); + + // mediaType.Charset for MVC + foreach (var type in mediaTypeList) { if (mediaType.IsSubsetOf(type)) @@ -211,85 +223,5 @@ private void FilterHeaders(List> keyValues, IHeade keyValues.Add(new KeyValuePair(key, value.ToString())); } } - - private async Task ReadRequestBody(HttpRequest request, CancellationToken token) - { - if (_options.BodyEncoding == null) - { - return "X"; - } - - using var joinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); - joinedTokenSource.CancelAfter(_options.RequestBodyTimeout); - var limit = _options.RequestBodyLogLimit; - - // Use a pipe for smaller payload sizes - if (limit <= PipeThreshold) - { - try - { - while (true) - { - // TODO if someone uses the body after this, it will not have the rest of the data. - var result = await request.BodyReader.ReadAsync(joinedTokenSource.Token); - if (!result.IsCompleted && result.Buffer.Length <= limit) - { - // Need more data. - request.BodyReader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - continue; - } - - var res = _options.BodyEncoding.GetString(result.Buffer.Slice(0, result.Buffer.Length > limit ? limit : result.Buffer.Length)); - - request.BodyReader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - return res; - } - } - catch (OperationCanceledException) - { - // Token source hides triggering token (https://github.com/dotnet/runtime/issues/22172) - if (!token.IsCancellationRequested && joinedTokenSource.Token.IsCancellationRequested) - { - return "X"; - } - - throw; - } - } - else - { - request.EnableBuffering(); - - // Read here. - var buffer = ArrayPool.Shared.Rent(limit); - - try - { - var count = 0; - while (true) - { - var read = await request.Body.ReadAsync(buffer, count, limit - count); - count += read; - - Debug.Assert(count <= limit); - if (read == 0 || count == limit) - { - break; - } - } - - return _options.BodyEncoding.GetString(new Span(buffer).Slice(0, count)); - } - finally - { - ArrayPool.Shared.Return(buffer); - // reseek back to start. - if (request.Body.CanSeek) - { - _ = request.Body.Seek(0, SeekOrigin.Begin); - } - } - } - } } } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index 100558f1222d..ddace9b7ce42 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -33,6 +33,7 @@ public sealed class HttpLoggingOptions HeaderNames.AcceptLanguage, HeaderNames.Allow, HeaderNames.Connection, + HeaderNames.ContentLength, HeaderNames.ContentType, HeaderNames.Host, HeaderNames.UserAgent @@ -47,10 +48,8 @@ public sealed class HttpLoggingOptions /// public HashSet AllowedResponseHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { + HeaderNames.ContentLength, HeaderNames.ContentType, - HeaderNames.Date, - HeaderNames.ETag, - HeaderNames.Server, HeaderNames.TransferEncoding }; @@ -78,11 +77,6 @@ public sealed class HttpLoggingOptions /// public int RequestBodyLogLimit { get; set; } = 32 * 1024; - /// - /// Timeout for reading requset body to log. - /// - public TimeSpan RequestBodyTimeout { get; set; } = Debugger.IsAttached ? TimeSpan.FromDays(1) : TimeSpan.FromSeconds(1); - /// /// Maximum response body size to log (in bytes). Defaults to 32 KB. /// diff --git a/src/Middleware/HttpLogging/src/LoggerEventIds.cs b/src/Middleware/HttpLogging/src/LoggerEventIds.cs index 83780d4b5599..ddc4f8e5e8c8 100644 --- a/src/Middleware/HttpLogging/src/LoggerEventIds.cs +++ b/src/Middleware/HttpLogging/src/LoggerEventIds.cs @@ -9,5 +9,7 @@ internal static class LoggerEventIds { public static readonly EventId RequestLog = new EventId(1, "RequestLog"); public static readonly EventId ResponseLog = new EventId(2, "ResponseLog"); + public static readonly EventId RequestBody = new EventId(3, "RequestBody"); + public static readonly EventId ResponseBody = new EventId(4, "ResponseBody"); } } diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index 417a8db2fae5..cfd3a528bd45 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -21,4 +21,5 @@ + diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index f35f6cce2272..274e9543f475 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -28,8 +28,6 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.get -> Microso Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.set -> void -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyTimeout.get -> System.TimeSpan -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyTimeout.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List? diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs new file mode 100644 index 000000000000..e47f056c024d --- /dev/null +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -0,0 +1,303 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal sealed class RequestBufferingStream : Stream, IBufferWriter + { + private const int MaxSegmentPoolSize = 256; // 1MB + private const int MinimumBufferSize = 4096; // 4K + private Stream _innerStream; + private Encoding? _encoding; + private readonly int _limit; + private readonly ILogger _logger; + private int _bytesWritten; + + private readonly BufferSegmentStack _bufferSegmentPool; + private BufferSegment? _head; + private BufferSegment? _tail; + private Memory _tailMemory; // remainder of tail memory + private int _tailBytesBuffered; + + public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) + { + _limit = limit; + _logger = logger; + _innerStream = innerStream; + _encoding = encoding; + _bufferSegmentPool = new BufferSegmentStack(limit / MinimumBufferSize); + } + + public override bool CanSeek => false; + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int WriteTimeout + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override async ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) + { + var res = await _innerStream.ReadAsync(destination, cancellationToken); + + WriteToBuffer(destination.Slice(0, res).Span, res); + + return res; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var res = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + + WriteToBuffer(buffer.AsSpan(offset, res), res); + + return res; + } + + public override int Read(byte[] buffer, int offset, int count) + { + var res = _innerStream.Read(buffer, offset, count); + + WriteToBuffer(buffer.AsSpan(offset, res), res); + + return res; + } + + private void WriteToBuffer(ReadOnlySpan span, int res) + { + // get what was read into the buffer + var remaining = _limit - _bytesWritten; + + if (remaining == 0) + { + return; + } + + if (res == 0) + { + LogString(); + } + + var innerCount = Math.Min(remaining, span.Length); + + if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) + { + _tailBytesBuffered += innerCount; + _bytesWritten += innerCount; + _tailMemory = _tailMemory.Slice(innerCount); + } + else + { + BuffersExtensions.Write(this, span.Slice(0, innerCount)); + } + + if (_limit - _bytesWritten == 0) + { + LogString(); + } + } + + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException(); + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) + { + return TaskToApm.Begin(ReadAsync(buffer, offset, count), callback, state); + } + + /// + public override int EndRead(IAsyncResult asyncResult) + { + return TaskToApm.End(asyncResult); + } + + /// + public override async Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + var remaining = _limit - _bytesWritten; + + while (remaining > 0) + { + // reusing inner buffer here between streams + var memory = GetMemory(); + var innerCount = Math.Min(remaining, memory.Length); + + var res = await _innerStream.ReadAsync(memory.Slice(0, innerCount), cancellationToken); + + _tailBytesBuffered += res; + _bytesWritten += res; + _tailMemory = _tailMemory.Slice(res); + + await destination.WriteAsync(memory.Slice(0, res)); + + remaining -= res; + } + + await _innerStream.CopyToAsync(destination, bufferSize, cancellationToken); + } + + public void LogString() + { + if (_head == null || _tail == null || _encoding == null) + { + return; + } + + // Only place where we are actually using the buffered data. + // update tail here. + _tail.End = _tailBytesBuffered; + + var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); + + // If encoding is nullable + var body = _encoding.GetString(ros); + _logger.LogInformation(LoggerEventIds.RequestBody, CoreStrings.RequestBody, body); + } + + public void Advance(int bytes) + { + if ((uint)bytes > (uint)_tailMemory.Length) + { + ThrowArgumentOutOfRangeException(nameof(bytes)); + } + + _tailBytesBuffered += bytes; + _bytesWritten += bytes; + _tailMemory = _tailMemory.Slice(bytes); + } + + public Memory GetMemory(int sizeHint = 0) + { + AllocateMemoryUnsynchronized(sizeHint); + return _tailMemory; + } + + public Span GetSpan(int sizeHint = 0) + { + AllocateMemoryUnsynchronized(sizeHint); + return _tailMemory.Span; + } + private void AllocateMemoryUnsynchronized(int sizeHint) + { + if (_head == null) + { + // We need to allocate memory to write since nobody has written before + BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + + // Set all the pointers + _head = _tail = newSegment; + _tailBytesBuffered = 0; + } + else + { + int bytesLeftInBuffer = _tailMemory.Length; + + if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < sizeHint) + { + Debug.Assert(_tail != null); + + if (_tailBytesBuffered > 0) + { + // Flush buffered data to the segment + _tail.End += _tailBytesBuffered; + _tailBytesBuffered = 0; + } + + BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + + _tail.SetNext(newSegment); + _tail = newSegment; + } + } + } + + private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) + { + BufferSegment newSegment = CreateSegmentUnsynchronized(); + + // We can't use the recommended pool so use the ArrayPool + newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); + + _tailMemory = newSegment.AvailableMemory; + + return newSegment; + } + + private BufferSegment CreateSegmentUnsynchronized() + { + if (_bufferSegmentPool.TryPop(out var segment)) + { + return segment; + } + + return new BufferSegment(); + } + + private void ReturnSegmentUnsynchronized(BufferSegment segment) + { + if (_bufferSegmentPool.Count < MaxSegmentPoolSize) + { + _bufferSegmentPool.Push(segment); + } + } + + private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) + { + // First we need to handle case where hint is smaller than minimum segment size + sizeHint = Math.Max(MinimumBufferSize, sizeHint); + // After that adjust it to fit into pools max buffer size + var adjustedToMaximumSize = Math.Min(maxBufferSize, sizeHint); + return adjustedToMaximumSize; + } + + // Copied from https://github.com/dotnet/corefx/blob/de3902bb56f1254ec1af4bf7d092fc2c048734cc/src/System.Memory/src/System/ThrowHelper.cs + private static void ThrowArgumentOutOfRangeException(string argumentName) { throw CreateArgumentOutOfRangeException(argumentName); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Exception CreateArgumentOutOfRangeException(string argumentName) { return new ArgumentOutOfRangeException(argumentName); } + } +} diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 8b978430fdf6..e6be5f224419 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -14,9 +14,13 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; +using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.HttpLogging { + /// + /// Stream that buffers reads + /// internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBufferWriter { private const int MaxSegmentPoolSize = 256; // 1MB @@ -30,6 +34,7 @@ internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBuff private int _bytesWritten; private readonly BufferSegmentStack _bufferSegmentPool; + private readonly ILogger _logger; private BufferSegment? _head; private BufferSegment? _tail; private Memory _tailMemory; // remainder of tail memory @@ -37,12 +42,13 @@ internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBuff private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true); - internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit) + internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit, ILogger logger) { _innerBodyFeature = innerBodyFeature; _innerStream = innerBodyFeature.Stream; _limit = limit; _bufferSegmentPool = new BufferSegmentStack(limit / MinimumBufferSize); + _logger = logger; } public override bool CanRead => false; @@ -178,6 +184,7 @@ public async Task CompleteAsync() // IBufferWriter public void Reset() { + _bytesWritten = 0; _tailBytesBuffered = 0; } @@ -245,7 +252,7 @@ private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) BufferSegment newSegment = CreateSegmentUnsynchronized(); // We can't use the recommended pool so use the ArrayPool - newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(Math.Max(MinimumBufferSize, sizeHint))); + newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); _tailMemory = newSegment.AvailableMemory; @@ -279,32 +286,6 @@ private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue return adjustedToMaximumSize; } - public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } - - public override void WriteByte(byte value) - { - if (_bytesWritten == _limit) - { - return; - } - - if (_tailMemory.Length == 0) - { - AllocateMemoryUnsynchronized(MinimumBufferSize); - } - - _tailMemory.Span[0] = value; - _tailBytesBuffered++; - _bytesWritten++; - _tailMemory = _tailMemory.Slice(1); - } - - // TODO inefficient, don't want to allocate array size of string, - // but issues with decoder APIs returning character count larger than - // required. public string GetString(Encoding? encoding) { if (_head == null || _tail == null || encoding == null) @@ -319,6 +300,8 @@ public string GetString(Encoding? encoding) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); // If encoding is nullable + + // TODO return buffers. return encoding.GetString(ros); } diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index fb9880223288..e3d10d2c6445 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -477,7 +477,7 @@ public async Task DifferentEncodingsWork() count += res; } - Assert.Equal(1000, count); + Assert.Equal(2000, count); }, options, LoggerFactory); @@ -491,80 +491,6 @@ public async Task DifferentEncodingsWork() Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); } - [Fact] - public async Task DefaultBodyTimeoutTruncates() - { - // media headers that should work. - var expected = new string('a', 1000); - var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; - - var middleware = new HttpLoggingMiddleware( - async c => - { - var arr = new byte[4096]; - var count = 0; - - while (true) - { - var res = await c.Request.Body.ReadAsync(arr); - if (res == 0) - { - break; - } - count += res; - } - - Assert.Equal(1000, count); - }, - options, - LoggerFactory); - - var httpContext = new DefaultHttpContext(); - httpContext.Request.ContentType = "text/plain"; - httpContext.Request.Body = new SlowStream(new MemoryStream(Encoding.ASCII.GetBytes("test")), TimeSpan.FromSeconds(2)); - - await Assert.ThrowsAsync(async () => await middleware.Invoke(httpContext)); - } - - [Fact] - public async Task CanSetTimeoutToDifferentValue() - { - // media headers that should work. - var expected = new string('a', 1000); - var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; - options.Value.RequestBodyTimeout = TimeSpan.FromSeconds(30); - - var middleware = new HttpLoggingMiddleware( - async c => - { - var arr = new byte[4096]; - var count = 0; - - while (true) - { - var res = await c.Request.Body.ReadAsync(arr); - if (res == 0) - { - break; - } - count += res; - } - - Assert.Equal(1000, count); - }, - options, - LoggerFactory); - - var httpContext = new DefaultHttpContext(); - httpContext.Request.ContentType = "text/plain"; - // Does not timeout. - httpContext.Request.Body = new SlowStream(new MemoryStream(Encoding.ASCII.GetBytes("test")), TimeSpan.FromSeconds(2)); - - await middleware.Invoke(httpContext); - } - [Fact] public async Task DefaultResponseInfoOnlyHeadersAndRequestInfo() { From 1760864d5a0a86823b169bc85a5c466523fb92ea Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 15 Apr 2021 22:38:43 -0700 Subject: [PATCH 12/45] Another API pass --- .../HttpLogging/src/CoreStrings.resx | 59 ++++++++------- .../HttpLogging/src/HttpLoggingMiddleware.cs | 74 ++++++++----------- .../HttpLogging/src/HttpLoggingOptions.cs | 18 ++--- .../HttpLogging/src/MediaTypeHelpers.cs | 65 ++++++++++++++++ .../HttpLogging/src/PublicAPI.Unshipped.txt | 4 +- .../HttpLogging/src/RequestBufferingStream.cs | 43 +++++++---- .../src/ResponseBufferingStream.cs | 62 +++++++++------- .../test/HttpLoggingMiddlewareTests.cs | 31 +++++--- 8 files changed, 221 insertions(+), 135 deletions(-) create mode 100644 src/Middleware/HttpLogging/src/MediaTypeHelpers.cs diff --git a/src/Middleware/HttpLogging/src/CoreStrings.resx b/src/Middleware/HttpLogging/src/CoreStrings.resx index 2a560a73a722..3f677a3bf28d 100644 --- a/src/Middleware/HttpLogging/src/CoreStrings.resx +++ b/src/Middleware/HttpLogging/src/CoreStrings.resx @@ -1,17 +1,17 @@  - @@ -120,4 +120,7 @@ RequestBody: {0} - \ No newline at end of file + + ResponseBody: {0} + + diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index b9108f688384..21ecaf756b1c 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -25,7 +25,6 @@ internal sealed class HttpLoggingMiddleware private readonly RequestDelegate _next; private readonly ILogger _logger; private HttpLoggingOptions _options; - private const int PipeThreshold = 32 * 1024; private const int DefaultRequestFieldsMinusHeaders = 7; private const int DefaultResponseFieldsMinusHeaders = 2; @@ -66,7 +65,8 @@ public async Task Invoke(HttpContext context) if ((HttpLoggingFields.Request & _options.LoggingFields) != HttpLoggingFields.None) { var request = context.Request; - var list = new List>(request.Headers.Count + DefaultRequestFieldsMinusHeaders); + var list = new List>( + request.Headers.Count + DefaultRequestFieldsMinusHeaders); if (_options.LoggingFields.HasFlag(HttpLoggingFields.Protocol)) { @@ -99,10 +99,17 @@ public async Task Invoke(HttpContext context) FilterHeaders(list, request.Headers, _options.AllowedRequestHeaders); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody)) + if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) + && MediaTypeHelpers.TryGetEncodingForMediaType(request.ContentType, + _options.SupportedMediaTypes, + out var encoding)) { originalBody = request.Body; - requestBufferingStream = new RequestBufferingStream(request.Body, _options.RequestBodyLogLimit, _logger, _options.BodyEncoding); + requestBufferingStream = new RequestBufferingStream( + request.Body, + _options.RequestBodyLogLimit, + _logger, + encoding); request.Body = requestBufferingStream; } @@ -134,12 +141,17 @@ public async Task Invoke(HttpContext context) { originalBodyFeature = context.Features.Get()!; // TODO pool these. - responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, _options.ResponseBodyLogLimit, _logger); + responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, + _options.ResponseBodyLogLimit, + _logger, + context, + _options.SupportedMediaTypes); response.Body = responseBufferingStream; } await _next(context).ConfigureAwait(false); - var list = new List>(response.Headers.Count + DefaultResponseFieldsMinusHeaders); + var list = new List>( + response.Headers.Count + DefaultResponseFieldsMinusHeaders); if (_options.LoggingFields.HasFlag(HttpLoggingFields.StatusCode)) { @@ -151,12 +163,6 @@ public async Task Invoke(HttpContext context) FilterHeaders(list, response.Headers, _options.AllowedResponseHeaders); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody) && IsSupportedMediaType(response.ContentType)) - { - var body = responseBufferingStream!.GetString(_options.BodyEncoding); - list.Add(new KeyValuePair(nameof(response.Body), body)); - } - var httpResponseLog = new HttpResponseLog(list); _logger.Log(LogLevel.Information, @@ -164,20 +170,25 @@ public async Task Invoke(HttpContext context) state: httpResponseLog, exception: null, formatter: HttpResponseLog.Callback); + if (responseBufferingStream != null) + { + responseBufferingStream.LogString(); + } } finally { - if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) - { - responseBufferingStream!.Dispose(); + responseBufferingStream?.Dispose(); + if (originalBodyFeature != null) + { context.Features.Set(originalBodyFeature); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody)) + requestBufferingStream?.Dispose(); + + if (originalBody != null) { - requestBufferingStream!.Dispose(); - context.Request.Body = originalBody!; + context.Request.Body = originalBody; } } } @@ -187,30 +198,9 @@ private static void AddToList(List> list, string k list.Add(new KeyValuePair(key, value)); } - private bool IsSupportedMediaType(string contentType) - { - var mediaTypeList = _options.SupportedMediaTypes; - if (mediaTypeList == null || mediaTypeList.Count == 0 || string.IsNullOrEmpty(contentType)) - { - return false; - } - - var mediaType = new MediaTypeHeaderValue(contentType); - - // mediaType.Charset for MVC - - foreach (var type in mediaTypeList) - { - if (mediaType.IsSubsetOf(type)) - { - return true; - } - } - - return false; - } - - private void FilterHeaders(List> keyValues, IHeaderDictionary headers, ISet allowedHeaders) + private void FilterHeaders(List> keyValues, + IHeaderDictionary headers, + ISet allowedHeaders) { foreach (var (key, value) in headers) { diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index ddace9b7ce42..738e9465c120 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -54,22 +54,16 @@ public sealed class HttpLoggingOptions }; /// - /// The encoding to log the request body and response body, if the content-type of the body - /// is one of the . - /// - public Encoding? BodyEncoding { get; set; } = Encoding.UTF8; - - /// - /// A list of supported media type values for request and response body logging. + /// A list of supported media type values for request and response body logging with corresponding encodings. /// /// /// If the request or response do not match the supported media type, the response body will not be logged. /// - public List? SupportedMediaTypes { get; } = new List() + public List> SupportedMediaTypes { get; } = new List>() { - new MediaTypeHeaderValue("application/json"), - new MediaTypeHeaderValue("application/xml"), - new MediaTypeHeaderValue("text/*") + new KeyValuePair(new MediaTypeHeaderValue("application/json"), Encoding.UTF8), + new KeyValuePair(new MediaTypeHeaderValue("application/xml"), Encoding.UTF8), + new KeyValuePair(new MediaTypeHeaderValue("text/*"), Encoding.UTF8), }; /// @@ -80,6 +74,6 @@ public sealed class HttpLoggingOptions /// /// Maximum response body size to log (in bytes). Defaults to 32 KB. /// - public int ResponseBodyLogLimit { get; set; } = 32 * 1024; // 32KB + public int ResponseBodyLogLimit { get; set; } = 32 * 1024; } } diff --git a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs new file mode 100644 index 000000000000..1f5c18281e34 --- /dev/null +++ b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Net.Http.Headers; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal static class MediaTypeHelpers + { + private static List SupportedEncodings = new List() + { + Encoding.UTF8, + Encoding.Unicode, + Encoding.ASCII, + Encoding.Latin1 // TODO allowed by default? Make this configurable? + }; + + public static bool TryGetEncodingForMediaType(string contentType, List> mediaTypeList, out Encoding? encoding) + { + encoding = null; + if (mediaTypeList == null || mediaTypeList.Count == 0 || string.IsNullOrEmpty(contentType)) + { + return false; + } + + var mediaType = new MediaTypeHeaderValue(contentType); + + if (mediaType.Charset.HasValue) + { + // Create encoding based on charset + var requestEncoding = mediaType.Encoding; + + if (requestEncoding != null) + { + for (var i = 0; i < SupportedEncodings.Count; i++) + { + if (string.Equals(requestEncoding.WebName, + SupportedEncodings[i].WebName, + StringComparison.OrdinalIgnoreCase)) + { + encoding = SupportedEncodings[i]; + return true; + } + } + } + } + else + { + foreach (var kvp in mediaTypeList) + { + var type = kvp.Key; + if (mediaType.IsSubsetOf(type)) + { + encoding = kvp.Value; + return true; + } + } + } + + return false; + } + } +} diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index 274e9543f475..bccef00ced59 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -21,8 +21,6 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode = 32 -> Microsoft. Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedRequestHeaders.get -> System.Collections.Generic.HashSet! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedResponseHeaders.get -> System.Collections.Generic.HashSet! -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.BodyEncoding.get -> System.Text.Encoding? -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.BodyEncoding.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.HttpLoggingOptions() -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.get -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.set -> void @@ -30,7 +28,7 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.get -> i Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> void -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List? +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List>! Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index e47f056c024d..c2ad8f02ad9c 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -16,7 +16,6 @@ namespace Microsoft.AspNetCore.HttpLogging { internal sealed class RequestBufferingStream : Stream, IBufferWriter { - private const int MaxSegmentPoolSize = 256; // 1MB private const int MinimumBufferSize = 4096; // 4K private Stream _innerStream; private Encoding? _encoding; @@ -24,7 +23,6 @@ internal sealed class RequestBufferingStream : Stream, IBufferWriter private readonly ILogger _logger; private int _bytesWritten; - private readonly BufferSegmentStack _bufferSegmentPool; private BufferSegment? _head; private BufferSegment? _tail; private Memory _tailMemory; // remainder of tail memory @@ -36,7 +34,6 @@ public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Enc _logger = logger; _innerStream = innerStream; _encoding = encoding; - _bufferSegmentPool = new BufferSegmentStack(limit / MinimumBufferSize); } public override bool CanSeek => false; @@ -98,6 +95,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (res == 0) { + // Done reading, log the string. LogString(); } @@ -197,6 +195,8 @@ public void LogString() // If encoding is nullable var body = _encoding.GetString(ros); _logger.LogInformation(LoggerEventIds.RequestBody, CoreStrings.RequestBody, body); + + Reset(); } public void Advance(int bytes) @@ -270,22 +270,9 @@ private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) private BufferSegment CreateSegmentUnsynchronized() { - if (_bufferSegmentPool.TryPop(out var segment)) - { - return segment; - } - return new BufferSegment(); } - private void ReturnSegmentUnsynchronized(BufferSegment segment) - { - if (_bufferSegmentPool.Count < MaxSegmentPoolSize) - { - _bufferSegmentPool.Push(segment); - } - } - private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) { // First we need to handle case where hint is smaller than minimum segment size @@ -295,6 +282,30 @@ private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue return adjustedToMaximumSize; } + protected override void Dispose(bool disposing) + { + if (disposing) + { + Reset(); + } + } + + public void Reset() + { + var segment = _head; + while (segment != null) + { + var returnSegment = segment; + segment = segment.NextSegment; + + // We haven't reached the tail of the linked list yet, so we can always return the returnSegment. + returnSegment.ResetMemory(); + } + + _bytesWritten = 0; + _tailBytesBuffered = 0; + } + // Copied from https://github.com/dotnet/corefx/blob/de3902bb56f1254ec1af4bf7d092fc2c048734cc/src/System.Memory/src/System/ThrowHelper.cs private static void ThrowArgumentOutOfRangeException(string argumentName) { throw CreateArgumentOutOfRangeException(argumentName); } [MethodImpl(MethodImplOptions.NoInlining)] diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index e6be5f224419..1b7321e46938 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -5,7 +5,6 @@ using System.Buffers; using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Pipelines; using System.Runtime.CompilerServices; @@ -15,6 +14,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; namespace Microsoft.AspNetCore.HttpLogging { @@ -23,7 +23,6 @@ namespace Microsoft.AspNetCore.HttpLogging /// internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBufferWriter { - private const int MaxSegmentPoolSize = 256; // 1MB private const int MinimumBufferSize = 4096; // 4K private readonly IHttpResponseBodyFeature _innerBodyFeature; @@ -33,22 +32,31 @@ internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBuff private int _bytesWritten; - private readonly BufferSegmentStack _bufferSegmentPool; private readonly ILogger _logger; + private readonly HttpContext _context; + private readonly List> _encodings; private BufferSegment? _head; private BufferSegment? _tail; private Memory _tailMemory; // remainder of tail memory private int _tailBytesBuffered; + private Encoding? _encoding; + private bool _hasCheckedEncoding; + private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true); - internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit, ILogger logger) + internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, + int limit, ILogger logger, + HttpContext context, + List> encodings) { + // TODO need first write event _innerBodyFeature = innerBodyFeature; _innerStream = innerBodyFeature.Stream; _limit = limit; - _bufferSegmentPool = new BufferSegmentStack(limit / MinimumBufferSize); _logger = logger; + _context = context; + _encodings = encodings; } public override bool CanRead => false; @@ -181,9 +189,17 @@ public async Task CompleteAsync() await _innerBodyFeature.CompleteAsync(); } - // IBufferWriter public void Reset() { + var segment = _head; + while (segment != null) + { + var returnSegment = segment; + segment = segment.NextSegment; + + // We haven't reached the tail of the linked list yet, so we can always return the returnSegment. + returnSegment.ResetMemory(); + } _bytesWritten = 0; _tailBytesBuffered = 0; @@ -249,6 +265,13 @@ private void AllocateMemoryUnsynchronized(int sizeHint) private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) { + if (!_hasCheckedEncoding) + { + MediaTypeHelpers.TryGetEncodingForMediaType(_context.Response.ContentType, _encodings, out _encoding); + _hasCheckedEncoding = true; + // TODO can short circuit this and not allocate anything if encoding is null after this. + } + BufferSegment newSegment = CreateSegmentUnsynchronized(); // We can't use the recommended pool so use the ArrayPool @@ -261,22 +284,9 @@ private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) private BufferSegment CreateSegmentUnsynchronized() { - if (_bufferSegmentPool.TryPop(out var segment)) - { - return segment; - } - return new BufferSegment(); } - private void ReturnSegmentUnsynchronized(BufferSegment segment) - { - if (_bufferSegmentPool.Count < MaxSegmentPoolSize) - { - _bufferSegmentPool.Push(segment); - } - } - private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) { // First we need to handle case where hint is smaller than minimum segment size @@ -286,11 +296,11 @@ private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue return adjustedToMaximumSize; } - public string GetString(Encoding? encoding) + public void LogString() { - if (_head == null || _tail == null || encoding == null) + if (_head == null || _tail == null || _encoding == null) { - return ""; + return; } // Only place where we are actually using the buffered data. @@ -299,10 +309,12 @@ public string GetString(Encoding? encoding) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); - // If encoding is nullable + var body = _encoding.GetString(ros); + + _logger.LogInformation(LoggerEventIds.ResponseBody, CoreStrings.ResponseBody, body); - // TODO return buffers. - return encoding.GetString(ros); + // Don't hold onto buffers anymore + Reset(); } protected override void Dispose(bool disposing) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index e3d10d2c6445..0b13fea1c677 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; @@ -10,6 +11,7 @@ using Microsoft.AspNetCore.HttpLogging; using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.Options; +using Microsoft.Net.Http.Headers; using Moq; using Xunit; @@ -128,9 +130,17 @@ public async Task RequestLogsAllRequestInfo() var options = CreateOptionsAccessor(); options.Value.LoggingFields = HttpLoggingFields.Request; var middleware = new HttpLoggingMiddleware( - c => + async c => { - return Task.CompletedTask; + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } }, options, LoggerFactory); @@ -460,7 +470,10 @@ public async Task DifferentEncodingsWork() var expected = new string('a', 1000); var options = CreateOptionsAccessor(); options.Value.LoggingFields = HttpLoggingFields.RequestBody; - options.Value.BodyEncoding = encoding; + options.Value.SupportedMediaTypes.Add( + new KeyValuePair( + new MediaTypeHeaderValue("text/plain"), + encoding)); var middleware = new HttpLoggingMiddleware( async c => @@ -498,7 +511,7 @@ public async Task DefaultResponseInfoOnlyHeadersAndRequestInfo() async c => { c.Response.StatusCode = 200; - c.Response.Headers["Server"] = "Kestrel"; + c.Response.Headers[HeaderNames.TransferEncoding] = "test"; c.Response.ContentType = "text/plain"; await c.Response.WriteAsync("test"); }, @@ -509,7 +522,7 @@ public async Task DefaultResponseInfoOnlyHeadersAndRequestInfo() await middleware.Invoke(httpContext); Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Transfer-Encoding: test")); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); } @@ -523,7 +536,7 @@ public async Task ResponseInfoLogsAll() async c => { c.Response.StatusCode = 200; - c.Response.Headers["Server"] = "Kestrel"; + c.Response.Headers[HeaderNames.TransferEncoding] = "test"; c.Response.ContentType = "text/plain"; await c.Response.WriteAsync("test"); }, @@ -534,7 +547,7 @@ public async Task ResponseInfoLogsAll() await middleware.Invoke(httpContext); Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Transfer-Encoding: test")); Assert.Contains(TestSink.Writes, w => w.Message.Contains("Body: test")); } @@ -574,7 +587,7 @@ public async Task ResponseHeadersLogs() async c => { c.Response.StatusCode = 200; - c.Response.Headers["Server"] = "Kestrel"; + c.Response.Headers[HeaderNames.TransferEncoding] = "test"; c.Response.ContentType = "text/plain"; await c.Response.WriteAsync("test"); }, @@ -585,7 +598,7 @@ public async Task ResponseHeadersLogs() await middleware.Invoke(httpContext); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: Kestrel")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Transfer-Encoding: test")); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); } From 28e267373ba76c8df26b5564098921d3dcb52146 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 08:48:09 -0700 Subject: [PATCH 13/45] Combine request and response stream base type --- .../HttpLogging/src/BufferingStream.cs | 202 ++++++++++++++++++ .../HttpLogging/src/HttpLoggingFields.cs | 10 +- .../HttpLogging/src/HttpLoggingMiddleware.cs | 3 +- .../HttpLogging/src/HttpLoggingOptions.cs | 2 + .../HttpLogging/src/RequestBufferingStream.cs | 147 +------------ .../src/ResponseBufferingStream.cs | 166 ++------------ .../test/HttpLoggingMiddlewareTests.cs | 1 + 7 files changed, 233 insertions(+), 298 deletions(-) create mode 100644 src/Middleware/HttpLogging/src/BufferingStream.cs diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs new file mode 100644 index 000000000000..7c8ada04fedc --- /dev/null +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -0,0 +1,202 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Pipelines; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.HttpLogging +{ + internal class BufferingStream : Stream, IBufferWriter + { + private const int MinimumBufferSize = 4096; // 4K + private readonly ILogger _logger; + private int _bytesWritten; + private BufferSegment? _head; + private BufferSegment? _tail; + protected Memory _tailMemory; // remainder of tail memory + protected int _tailBytesBuffered; + + public BufferingStream(ILogger logger) + { + _logger = logger; + } + + public override bool CanRead => throw new NotImplementedException(); + + public override bool CanSeek => throw new NotImplementedException(); + + public override bool CanWrite => throw new NotImplementedException(); + + public override long Length => throw new NotImplementedException(); + + public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + + public void LogString(Encoding? encoding) + { + if (_head == null || _tail == null || encoding == null) + { + return; + } + + // Only place where we are actually using the buffered data. + // update tail here. + _tail.End = _tailBytesBuffered; + + var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); + + // If encoding is nullable + var body = encoding.GetString(ros); + _logger.LogInformation(LoggerEventIds.RequestBody, CoreStrings.RequestBody, body); + + Reset(); + } + + public void Advance(int bytes) + { + if ((uint)bytes > (uint)_tailMemory.Length) + { + ThrowArgumentOutOfRangeException(nameof(bytes)); + } + + _tailBytesBuffered += bytes; + _bytesWritten += bytes; + _tailMemory = _tailMemory.Slice(bytes); + } + + public Memory GetMemory(int sizeHint = 0) + { + AllocateMemoryUnsynchronized(sizeHint); + return _tailMemory; + } + + public Span GetSpan(int sizeHint = 0) + { + AllocateMemoryUnsynchronized(sizeHint); + return _tailMemory.Span; + } + private void AllocateMemoryUnsynchronized(int sizeHint) + { + if (_head == null) + { + // We need to allocate memory to write since nobody has written before + BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + + // Set all the pointers + _head = _tail = newSegment; + _tailBytesBuffered = 0; + } + else + { + int bytesLeftInBuffer = _tailMemory.Length; + + if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < sizeHint) + { + Debug.Assert(_tail != null); + + if (_tailBytesBuffered > 0) + { + // Flush buffered data to the segment + _tail.End += _tailBytesBuffered; + _tailBytesBuffered = 0; + } + + BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + + _tail.SetNext(newSegment); + _tail = newSegment; + } + } + } + + private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) + { + BufferSegment newSegment = CreateSegmentUnsynchronized(); + + // We can't use the recommended pool so use the ArrayPool + newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); + + _tailMemory = newSegment.AvailableMemory; + + return newSegment; + } + + private BufferSegment CreateSegmentUnsynchronized() + { + return new BufferSegment(); + } + + private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) + { + // First we need to handle case where hint is smaller than minimum segment size + sizeHint = Math.Max(MinimumBufferSize, sizeHint); + // After that adjust it to fit into pools max buffer size + var adjustedToMaximumSize = Math.Min(maxBufferSize, sizeHint); + return adjustedToMaximumSize; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + Reset(); + } + } + + public void Reset() + { + var segment = _head; + while (segment != null) + { + var returnSegment = segment; + segment = segment.NextSegment; + + // We haven't reached the tail of the linked list yet, so we can always return the returnSegment. + returnSegment.ResetMemory(); + } + + _head = _tail = null; + + _bytesWritten = 0; + _tailBytesBuffered = 0; + } + + // Copied from https://github.com/dotnet/corefx/blob/de3902bb56f1254ec1af4bf7d092fc2c048734cc/src/System.Memory/src/System/ThrowHelper.cs + private static void ThrowArgumentOutOfRangeException(string argumentName) { throw CreateArgumentOutOfRangeException(argumentName); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Exception CreateArgumentOutOfRangeException(string argumentName) { return new ArgumentOutOfRangeException(argumentName); } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index bb66ba1a0d1b..b9a91937affe 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -14,27 +14,27 @@ public enum HttpLoggingFields : long /// /// None /// - None = 0, + None = 0x0, /// /// Request Path /// - Path = 1, + Path = 0x1, /// /// Request Query String /// - Query = 2, + Query = 0x2, /// /// Request Protocol /// - Protocol = 4, + Protocol = 0x4, /// /// Request Method /// - Method = 8, + Method = 0x8, /// /// Request Scheme diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 21ecaf756b1c..9aac342e0716 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -170,9 +170,10 @@ public async Task Invoke(HttpContext context) state: httpResponseLog, exception: null, formatter: HttpResponseLog.Callback); + if (responseBufferingStream != null) { - responseBufferingStream.LogString(); + responseBufferingStream.LogString(responseBufferingStream.Encoding); } } finally diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index 738e9465c120..3454e3853f9e 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -62,7 +62,9 @@ public sealed class HttpLoggingOptions public List> SupportedMediaTypes { get; } = new List>() { new KeyValuePair(new MediaTypeHeaderValue("application/json"), Encoding.UTF8), + new KeyValuePair(new MediaTypeHeaderValue("application/*+json"), Encoding.UTF8), new KeyValuePair(new MediaTypeHeaderValue("application/xml"), Encoding.UTF8), + new KeyValuePair(new MediaTypeHeaderValue("application/*+xml"), Encoding.UTF8), new KeyValuePair(new MediaTypeHeaderValue("text/*"), Encoding.UTF8), }; diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index c2ad8f02ad9c..193fb9fa13a7 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -14,24 +14,17 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal sealed class RequestBufferingStream : Stream, IBufferWriter + internal class RequestBufferingStream : BufferingStream { - private const int MinimumBufferSize = 4096; // 4K private Stream _innerStream; private Encoding? _encoding; private readonly int _limit; - private readonly ILogger _logger; private int _bytesWritten; - private BufferSegment? _head; - private BufferSegment? _tail; - private Memory _tailMemory; // remainder of tail memory - private int _tailBytesBuffered; - public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) + : base(logger) { _limit = limit; - _logger = logger; _innerStream = innerStream; _encoding = encoding; } @@ -96,7 +89,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (res == 0) { // Done reading, log the string. - LogString(); + LogString(_encoding); } var innerCount = Math.Min(remaining, span.Length); @@ -114,7 +107,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (_limit - _bytesWritten == 0) { - LogString(); + LogString(_encoding); } } @@ -178,137 +171,5 @@ public override async Task CopyToAsync(Stream destination, int bufferSize, Cance await _innerStream.CopyToAsync(destination, bufferSize, cancellationToken); } - - public void LogString() - { - if (_head == null || _tail == null || _encoding == null) - { - return; - } - - // Only place where we are actually using the buffered data. - // update tail here. - _tail.End = _tailBytesBuffered; - - var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); - - // If encoding is nullable - var body = _encoding.GetString(ros); - _logger.LogInformation(LoggerEventIds.RequestBody, CoreStrings.RequestBody, body); - - Reset(); - } - - public void Advance(int bytes) - { - if ((uint)bytes > (uint)_tailMemory.Length) - { - ThrowArgumentOutOfRangeException(nameof(bytes)); - } - - _tailBytesBuffered += bytes; - _bytesWritten += bytes; - _tailMemory = _tailMemory.Slice(bytes); - } - - public Memory GetMemory(int sizeHint = 0) - { - AllocateMemoryUnsynchronized(sizeHint); - return _tailMemory; - } - - public Span GetSpan(int sizeHint = 0) - { - AllocateMemoryUnsynchronized(sizeHint); - return _tailMemory.Span; - } - private void AllocateMemoryUnsynchronized(int sizeHint) - { - if (_head == null) - { - // We need to allocate memory to write since nobody has written before - BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); - - // Set all the pointers - _head = _tail = newSegment; - _tailBytesBuffered = 0; - } - else - { - int bytesLeftInBuffer = _tailMemory.Length; - - if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < sizeHint) - { - Debug.Assert(_tail != null); - - if (_tailBytesBuffered > 0) - { - // Flush buffered data to the segment - _tail.End += _tailBytesBuffered; - _tailBytesBuffered = 0; - } - - BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); - - _tail.SetNext(newSegment); - _tail = newSegment; - } - } - } - - private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) - { - BufferSegment newSegment = CreateSegmentUnsynchronized(); - - // We can't use the recommended pool so use the ArrayPool - newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); - - _tailMemory = newSegment.AvailableMemory; - - return newSegment; - } - - private BufferSegment CreateSegmentUnsynchronized() - { - return new BufferSegment(); - } - - private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) - { - // First we need to handle case where hint is smaller than minimum segment size - sizeHint = Math.Max(MinimumBufferSize, sizeHint); - // After that adjust it to fit into pools max buffer size - var adjustedToMaximumSize = Math.Min(maxBufferSize, sizeHint); - return adjustedToMaximumSize; - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - Reset(); - } - } - - public void Reset() - { - var segment = _head; - while (segment != null) - { - var returnSegment = segment; - segment = segment.NextSegment; - - // We haven't reached the tail of the linked list yet, so we can always return the returnSegment. - returnSegment.ResetMemory(); - } - - _bytesWritten = 0; - _tailBytesBuffered = 0; - } - - // Copied from https://github.com/dotnet/corefx/blob/de3902bb56f1254ec1af4bf7d092fc2c048734cc/src/System.Memory/src/System/ThrowHelper.cs - private static void ThrowArgumentOutOfRangeException(string argumentName) { throw CreateArgumentOutOfRangeException(argumentName); } - [MethodImpl(MethodImplOptions.NoInlining)] - private static Exception CreateArgumentOutOfRangeException(string argumentName) { return new ArgumentOutOfRangeException(argumentName); } } } diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 1b7321e46938..5595b2501631 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -21,7 +21,7 @@ namespace Microsoft.AspNetCore.HttpLogging /// /// Stream that buffers reads /// - internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBufferWriter + internal class ResponseBufferingStream : BufferingStream, IHttpResponseBodyFeature { private const int MinimumBufferSize = 4096; // 4K @@ -32,13 +32,8 @@ internal class ResponseBufferingStream : Stream, IHttpResponseBodyFeature, IBuff private int _bytesWritten; - private readonly ILogger _logger; private readonly HttpContext _context; private readonly List> _encodings; - private BufferSegment? _head; - private BufferSegment? _tail; - private Memory _tailMemory; // remainder of tail memory - private int _tailBytesBuffered; private Encoding? _encoding; private bool _hasCheckedEncoding; @@ -49,12 +44,12 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit, ILogger logger, HttpContext context, List> encodings) + : base (logger) { // TODO need first write event _innerBodyFeature = innerBodyFeature; _innerStream = innerBodyFeature.Stream; _limit = limit; - _logger = logger; _context = context; _encodings = encodings; } @@ -89,6 +84,8 @@ public PipeWriter Writer } } + public Encoding? Encoding { get => _encoding; } + public override int Read(byte[] buffer, int offset, int count) { throw new NotSupportedException(); @@ -134,6 +131,8 @@ public override void Write(ReadOnlySpan span) var remaining = _limit - _bytesWritten; var innerCount = Math.Min(remaining, span.Length); + OnFirstWrite(); + if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) { _tailBytesBuffered += innerCount; @@ -153,9 +152,10 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc var remaining = _limit - _bytesWritten; var innerCount = Math.Min(remaining, count); + OnFirstWrite(); + if (_tailMemory.Length - innerCount > 0) { - //Buffer.BlockCopy(buffer, offset, , position, innerCount); buffer.AsSpan(offset, count).CopyTo(_tailMemory.Span); _tailBytesBuffered += innerCount; _bytesWritten += innerCount; @@ -169,165 +169,33 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc await _innerStream.WriteAsync(buffer, offset, count, cancellationToken); } - public void DisableBuffering() - { - _innerBodyFeature.DisableBuffering(); - } - - public Task SendFileAsync(string path, long offset, long? count, CancellationToken cancellation) - { - return _innerBodyFeature.SendFileAsync(path, offset, count, cancellation); - } - - public Task StartAsync(CancellationToken token = default) - { - return _innerBodyFeature.StartAsync(token); - } - - public async Task CompleteAsync() - { - await _innerBodyFeature.CompleteAsync(); - } - - public void Reset() - { - var segment = _head; - while (segment != null) - { - var returnSegment = segment; - segment = segment.NextSegment; - - // We haven't reached the tail of the linked list yet, so we can always return the returnSegment. - returnSegment.ResetMemory(); - } - - _bytesWritten = 0; - _tailBytesBuffered = 0; - } - - public void Advance(int bytes) - { - if ((uint)bytes > (uint)_tailMemory.Length) - { - ThrowArgumentOutOfRangeException(nameof(bytes)); - } - - _tailBytesBuffered += bytes; - _bytesWritten += bytes; - _tailMemory = _tailMemory.Slice(bytes); - } - - public Memory GetMemory(int sizeHint = 0) - { - AllocateMemoryUnsynchronized(sizeHint); - return _tailMemory; - } - - public Span GetSpan(int sizeHint = 0) - { - AllocateMemoryUnsynchronized(sizeHint); - return _tailMemory.Span; - } - - private void AllocateMemoryUnsynchronized(int sizeHint) - { - if (_head == null) - { - // We need to allocate memory to write since nobody has written before - BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); - - // Set all the pointers - _head = _tail = newSegment; - _tailBytesBuffered = 0; - } - else - { - int bytesLeftInBuffer = _tailMemory.Length; - - if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < sizeHint) - { - Debug.Assert(_tail != null); - - if (_tailBytesBuffered > 0) - { - // Flush buffered data to the segment - _tail.End += _tailBytesBuffered; - _tailBytesBuffered = 0; - } - - BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); - - _tail.SetNext(newSegment); - _tail = newSegment; - } - } - } - - private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) + private void OnFirstWrite() { if (!_hasCheckedEncoding) { MediaTypeHelpers.TryGetEncodingForMediaType(_context.Response.ContentType, _encodings, out _encoding); _hasCheckedEncoding = true; - // TODO can short circuit this and not allocate anything if encoding is null after this. } - - BufferSegment newSegment = CreateSegmentUnsynchronized(); - - // We can't use the recommended pool so use the ArrayPool - newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); - - _tailMemory = newSegment.AvailableMemory; - - return newSegment; } - private BufferSegment CreateSegmentUnsynchronized() + public void DisableBuffering() { - return new BufferSegment(); + _innerBodyFeature.DisableBuffering(); } - private static int GetSegmentSize(int sizeHint, int maxBufferSize = int.MaxValue) + public Task SendFileAsync(string path, long offset, long? count, CancellationToken cancellation) { - // First we need to handle case where hint is smaller than minimum segment size - sizeHint = Math.Max(MinimumBufferSize, sizeHint); - // After that adjust it to fit into pools max buffer size - var adjustedToMaximumSize = Math.Min(maxBufferSize, sizeHint); - return adjustedToMaximumSize; + return _innerBodyFeature.SendFileAsync(path, offset, count, cancellation); } - public void LogString() + public Task StartAsync(CancellationToken token = default) { - if (_head == null || _tail == null || _encoding == null) - { - return; - } - - // Only place where we are actually using the buffered data. - // update tail here. - _tail.End = _tailBytesBuffered; - - var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); - - var body = _encoding.GetString(ros); - - _logger.LogInformation(LoggerEventIds.ResponseBody, CoreStrings.ResponseBody, body); - - // Don't hold onto buffers anymore - Reset(); + return _innerBodyFeature.StartAsync(token); } - protected override void Dispose(bool disposing) + public async Task CompleteAsync() { - if (disposing) - { - Reset(); - } + await _innerBodyFeature.CompleteAsync(); } - - // Copied from https://github.com/dotnet/corefx/blob/de3902bb56f1254ec1af4bf7d092fc2c048734cc/src/System.Memory/src/System/ThrowHelper.cs - private static void ThrowArgumentOutOfRangeException(string argumentName) { throw CreateArgumentOutOfRangeException(argumentName); } - [MethodImpl(MethodImplOptions.NoInlining)] - private static Exception CreateArgumentOutOfRangeException(string argumentName) { return new ArgumentOutOfRangeException(argumentName); } } } diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 0b13fea1c677..0c5c39ad11c9 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -470,6 +470,7 @@ public async Task DifferentEncodingsWork() var expected = new string('a', 1000); var options = CreateOptionsAccessor(); options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.Value.SupportedMediaTypes.Clear(); options.Value.SupportedMediaTypes.Add( new KeyValuePair( new MediaTypeHeaderValue("text/plain"), From 472f1b565e59fb3877b49f933e1bc2ad6deb5307 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 08:50:27 -0700 Subject: [PATCH 14/45] Fixing variable --- src/Middleware/HttpLogging/src/BufferingStream.cs | 2 +- src/Middleware/HttpLogging/src/RequestBufferingStream.cs | 1 - src/Middleware/HttpLogging/src/ResponseBufferingStream.cs | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 7c8ada04fedc..1a461a4411ff 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -19,7 +19,7 @@ internal class BufferingStream : Stream, IBufferWriter { private const int MinimumBufferSize = 4096; // 4K private readonly ILogger _logger; - private int _bytesWritten; + protected int _bytesWritten; private BufferSegment? _head; private BufferSegment? _tail; protected Memory _tailMemory; // remainder of tail memory diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index 193fb9fa13a7..e346d0dab040 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -19,7 +19,6 @@ internal class RequestBufferingStream : BufferingStream private Stream _innerStream; private Encoding? _encoding; private readonly int _limit; - private int _bytesWritten; public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) : base(logger) diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 5595b2501631..743b39f98ed0 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -30,8 +30,6 @@ internal class ResponseBufferingStream : BufferingStream, IHttpResponseBodyFeatu private readonly int _limit; private PipeWriter? _pipeAdapter; - private int _bytesWritten; - private readonly HttpContext _context; private readonly List> _encodings; From 9ac780689aa8e4db6f639f9e43c21a057097ad98 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 08:55:49 -0700 Subject: [PATCH 15/45] nit --- src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs | 1 - src/Middleware/HttpLogging/src/HttpResponseLog.cs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 9aac342e0716..fed889fbfee2 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -35,7 +35,6 @@ internal sealed class HttpLoggingMiddleware /// /// public HttpLoggingMiddleware(RequestDelegate next, IOptions options, ILoggerFactory loggerFactory) - { _next = next ?? throw new ArgumentNullException(nameof(next)); diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs index 945afc2fb6af..e7d0a6791366 100644 --- a/src/Middleware/HttpLogging/src/HttpResponseLog.cs +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -33,7 +33,8 @@ public HttpResponseLog(List> keyValues) } public override string ToString() - {if (_cachedToString == null) + { + if (_cachedToString == null) { var builder = new StringBuilder(); var count = _keyValues.Count; From 8f75b8a89a0aa579932986bcb3b889600875c7d5 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 09:11:50 -0700 Subject: [PATCH 16/45] Updating samples --- .../samples/HttpLogging.Sample/Program.cs | 13 ++++++ .../HttpLogging/src/HttpLoggingMiddleware.cs | 45 ++++++++++--------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs index 3eeae05dbf69..c4c215d505a8 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; namespace HttpLogging.Sample @@ -18,6 +19,18 @@ public static void Main(string[] args) public static IHostBuilder CreateHostBuilder(string[] args) => Host.CreateDefaultBuilder(args) + .ConfigureLogging(logging => + { + // Json Logging + logging.ClearProviders(); + logging.AddJsonConsole(options => + { + options.JsonWriterOptions = new JsonWriterOptions() + { + Indented = true + }; + }); + }) .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index fed889fbfee2..2dbc121096d6 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -24,7 +24,7 @@ internal sealed class HttpLoggingMiddleware { private readonly RequestDelegate _next; private readonly ILogger _logger; - private HttpLoggingOptions _options; + private IOptionsMonitor _options; private const int DefaultRequestFieldsMinusHeaders = 7; private const int DefaultResponseFieldsMinusHeaders = 2; @@ -34,7 +34,7 @@ internal sealed class HttpLoggingMiddleware /// /// /// - public HttpLoggingMiddleware(RequestDelegate next, IOptions options, ILoggerFactory loggerFactory) + public HttpLoggingMiddleware(RequestDelegate next, IOptionsMonitor options, ILoggerFactory loggerFactory) { _next = next ?? throw new ArgumentNullException(nameof(next)); @@ -48,7 +48,7 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptions throw new ArgumentNullException(nameof(loggerFactory)); } - _options = options.Value; + _options = options; _logger = loggerFactory.CreateLogger(); } @@ -56,57 +56,58 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptions /// Invokes the . /// /// - /// + /// HttpResponseLog.cs public async Task Invoke(HttpContext context) { + var options = _options.CurrentValue; RequestBufferingStream? requestBufferingStream = null; Stream? originalBody = null; - if ((HttpLoggingFields.Request & _options.LoggingFields) != HttpLoggingFields.None) + if ((HttpLoggingFields.Request & options.LoggingFields) != HttpLoggingFields.None) { var request = context.Request; var list = new List>( request.Headers.Count + DefaultRequestFieldsMinusHeaders); - if (_options.LoggingFields.HasFlag(HttpLoggingFields.Protocol)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.Protocol)) { AddToList(list, nameof(request.Protocol), request.Protocol); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.Method)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.Method)) { AddToList(list, nameof(request.Method), request.Method); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.Scheme)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.Scheme)) { AddToList(list, nameof(request.Scheme), request.Scheme); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.Path)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.Path)) { AddToList(list, nameof(request.PathBase), request.PathBase); AddToList(list, nameof(request.Path), request.Path); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.Query)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.Query)) { AddToList(list, nameof(request.QueryString), request.QueryString.Value); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders)) { - FilterHeaders(list, request.Headers, _options.AllowedRequestHeaders); + FilterHeaders(list, request.Headers, options.AllowedRequestHeaders); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) && MediaTypeHelpers.TryGetEncodingForMediaType(request.ContentType, - _options.SupportedMediaTypes, + options.SupportedMediaTypes, out var encoding)) { originalBody = request.Body; requestBufferingStream = new RequestBufferingStream( request.Body, - _options.RequestBodyLogLimit, + options.RequestBodyLogLimit, _logger, encoding); request.Body = requestBufferingStream; @@ -127,7 +128,7 @@ public async Task Invoke(HttpContext context) try { - if ((HttpLoggingFields.Response & _options.LoggingFields) == HttpLoggingFields.None) + if ((HttpLoggingFields.Response & options.LoggingFields) == HttpLoggingFields.None) { // Short circuit and don't replace response body. await _next(context).ConfigureAwait(false); @@ -136,15 +137,15 @@ public async Task Invoke(HttpContext context) var response = context.Response; - if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) { originalBodyFeature = context.Features.Get()!; // TODO pool these. responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, - _options.ResponseBodyLogLimit, + options.ResponseBodyLogLimit, _logger, context, - _options.SupportedMediaTypes); + options.SupportedMediaTypes); response.Body = responseBufferingStream; } @@ -152,14 +153,14 @@ public async Task Invoke(HttpContext context) var list = new List>( response.Headers.Count + DefaultResponseFieldsMinusHeaders); - if (_options.LoggingFields.HasFlag(HttpLoggingFields.StatusCode)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.StatusCode)) { list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); } - if (_options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) { - FilterHeaders(list, response.Headers, _options.AllowedResponseHeaders); + FilterHeaders(list, response.Headers, options.AllowedResponseHeaders); } var httpResponseLog = new HttpResponseLog(list); From 01efac0ff8bdebb20c2fd109ee741146ef38dcbe Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 09:43:01 -0700 Subject: [PATCH 17/45] Some feedback --- .../HttpLogging/src/HttpRequestLog.cs | 2 + .../HttpLogging/src/HttpResponseLog.cs | 2 + .../test/HttpLoggingMiddlewareTests.cs | 56 +++++++++---------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs index e6e09e0684cf..416f814cd3d9 100644 --- a/src/Middleware/HttpLogging/src/HttpRequestLog.cs +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -38,6 +38,8 @@ public override string ToString() { var builder = new StringBuilder(); var count = _keyValues.Count; + builder.Append("Request:"); + builder.Append(Environment.NewLine); for (var i = 0; i < count - 1; i++) { diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs index e7d0a6791366..0a9ec29b8825 100644 --- a/src/Middleware/HttpLogging/src/HttpResponseLog.cs +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -38,6 +38,8 @@ public override string ToString() { var builder = new StringBuilder(); var count = _keyValues.Count; + builder.Append("Response:"); + builder.Append(Environment.NewLine); for (var i = 0; i < count - 1; i++) { diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 0c5c39ad11c9..1d1c16238d01 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -56,7 +56,7 @@ public void Ctor_ThrowsExceptionsWhenNullArgs() public async Task NoopWhenLoggingDisabled() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.None; + options.CurrentValue.LoggingFields = HttpLoggingFields.None; var middleware = new HttpLoggingMiddleware( c => @@ -128,7 +128,7 @@ public async Task DefaultRequestInfoOnlyHeadersAndRequestInfo() public async Task RequestLogsAllRequestInfo() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.Request; + options.CurrentValue.LoggingFields = HttpLoggingFields.Request; var middleware = new HttpLoggingMiddleware( async c => { @@ -171,7 +171,7 @@ public async Task RequestLogsAllRequestInfo() public async Task RequestPropertiesLogs() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestProperties; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestProperties; var middleware = new HttpLoggingMiddleware( c => { @@ -206,7 +206,7 @@ public async Task RequestPropertiesLogs() public async Task RequestHeadersLogs() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestHeaders; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestHeaders; var middleware = new HttpLoggingMiddleware( c => { @@ -260,8 +260,8 @@ public async Task UnknownRequestHeadersRedacted() public async Task CanConfigureRequestAllowList() { var options = CreateOptionsAccessor(); - options.Value.AllowedRequestHeaders.Clear(); - options.Value.AllowedRequestHeaders.Add("foo"); + options.CurrentValue.AllowedRequestHeaders.Clear(); + options.CurrentValue.AllowedRequestHeaders.Add("foo"); var middleware = new HttpLoggingMiddleware( c => { @@ -287,7 +287,7 @@ public async Task CanConfigureRequestAllowList() public async Task RequestBodyReadingWorks(string expected) { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; var middleware = new HttpLoggingMiddleware( async c => @@ -319,7 +319,7 @@ public async Task RequestBodyReadingLimitWorks() { var input = string.Concat(new string('a', 30000), new string('b', 3000)); var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; var middleware = new HttpLoggingMiddleware( async c => @@ -342,7 +342,7 @@ public async Task RequestBodyReadingLimitWorks() httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); await middleware.Invoke(httpContext); - var expected = input.Substring(0, options.Value.ResponseBodyLogLimit); + var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit); Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); } @@ -352,7 +352,7 @@ public async Task RequestBodyReadingLimitGreaterThanPipeWorks() { var input = string.Concat(new string('a', 60000), new string('b', 3000)); var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; var middleware = new HttpLoggingMiddleware( async c => @@ -379,7 +379,7 @@ public async Task RequestBodyReadingLimitGreaterThanPipeWorks() httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); await middleware.Invoke(httpContext); - var expected = input.Substring(0, options.Value.ResponseBodyLogLimit); + var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit); Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); } @@ -396,7 +396,7 @@ public async Task VerifyDefaultMediaTypeHeaders(string contentType) // media headers that should work. var expected = new string('a', 1000); var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; var middleware = new HttpLoggingMiddleware( async c => @@ -431,7 +431,7 @@ public async Task RejectedContentTypes(string contentType) // media headers that should work. var expected = new string('a', 1000); var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; var middleware = new HttpLoggingMiddleware( async c => @@ -469,9 +469,9 @@ public async Task DifferentEncodingsWork() var encoding = Encoding.Unicode; var expected = new string('a', 1000); var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.RequestBody; - options.Value.SupportedMediaTypes.Clear(); - options.Value.SupportedMediaTypes.Add( + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.SupportedMediaTypes.Clear(); + options.CurrentValue.SupportedMediaTypes.Add( new KeyValuePair( new MediaTypeHeaderValue("text/plain"), encoding)); @@ -531,7 +531,7 @@ public async Task DefaultResponseInfoOnlyHeadersAndRequestInfo() public async Task ResponseInfoLogsAll() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.Response; + options.CurrentValue.LoggingFields = HttpLoggingFields.Response; var middleware = new HttpLoggingMiddleware( async c => @@ -557,7 +557,7 @@ public async Task ResponseInfoLogsAll() public async Task StatusCodeLogs() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.StatusCode; + options.CurrentValue.LoggingFields = HttpLoggingFields.StatusCode; var middleware = new HttpLoggingMiddleware( async c => @@ -582,7 +582,7 @@ public async Task StatusCodeLogs() public async Task ResponseHeadersLogs() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.ResponseHeaders; + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseHeaders; var middleware = new HttpLoggingMiddleware( async c => @@ -607,7 +607,7 @@ public async Task ResponseHeadersLogs() public async Task ResponseHeadersRedacted() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.ResponseHeaders; + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseHeaders; var middleware = new HttpLoggingMiddleware( c => @@ -628,9 +628,9 @@ public async Task ResponseHeadersRedacted() public async Task AllowedResponseHeadersModify() { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.ResponseHeaders; - options.Value.AllowedResponseHeaders.Clear(); - options.Value.AllowedResponseHeaders.Add("Test"); + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseHeaders; + options.CurrentValue.AllowedResponseHeaders.Clear(); + options.CurrentValue.AllowedResponseHeaders.Add("Test"); var middleware = new HttpLoggingMiddleware( c => @@ -654,7 +654,7 @@ public async Task AllowedResponseHeadersModify() public async Task ResponseBodyWritingWorks(string expected) { var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.ResponseBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseBody; var middleware = new HttpLoggingMiddleware( c => { @@ -676,7 +676,7 @@ public async Task ResponseBodyWritingLimitWorks() { var input = string.Concat(new string('a', 30000), new string('b', 3000)); var options = CreateOptionsAccessor(); - options.Value.LoggingFields = HttpLoggingFields.ResponseBody; + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseBody; var middleware = new HttpLoggingMiddleware( c => { @@ -690,14 +690,14 @@ public async Task ResponseBodyWritingLimitWorks() await middleware.Invoke(httpContext); - var expected = input.Substring(0, options.Value.ResponseBodyLogLimit); + var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit); Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); } - private IOptions CreateOptionsAccessor() + private IOptionsMonitor CreateOptionsAccessor() { var options = new HttpLoggingOptions(); - var optionsAccessor = Mock.Of>(o => o.Value == options); + var optionsAccessor = Mock.Of>(o => o.CurrentValue == options); return optionsAccessor; } From 47a2f0f662f4ff69a1e55b9188d5fd26840c41d9 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 11:27:26 -0700 Subject: [PATCH 18/45] Small fixups --- src/Middleware/HttpLogging/src/BufferingStream.cs | 4 ++-- .../HttpLogging/src/HttpLoggingMiddleware.cs | 2 +- src/Middleware/HttpLogging/src/HttpLoggingOptions.cs | 1 - .../HttpLogging/src/RequestBufferingStream.cs | 12 ++++++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 1a461a4411ff..36eb4c51a647 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -40,7 +40,7 @@ public BufferingStream(ILogger logger) public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - public void LogString(Encoding? encoding) + public void LogString(Encoding? encoding, EventId eventId, string template) { if (_head == null || _tail == null || encoding == null) { @@ -55,7 +55,7 @@ public void LogString(Encoding? encoding) // If encoding is nullable var body = encoding.GetString(ros); - _logger.LogInformation(LoggerEventIds.RequestBody, CoreStrings.RequestBody, body); + _logger.LogInformation(eventId, template, body); Reset(); } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 2dbc121096d6..3af379986163 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -173,7 +173,7 @@ public async Task Invoke(HttpContext context) if (responseBufferingStream != null) { - responseBufferingStream.LogString(responseBufferingStream.Encoding); + responseBufferingStream.LogString(responseBufferingStream.Encoding, LoggerEventIds.ResponseBody, CoreStrings.ResponseBody); } } finally diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index 3454e3853f9e..8b3c8f0ddabc 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Text; using Microsoft.Net.Http.Headers; diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index e346d0dab040..ccc8721e70d0 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -19,6 +19,7 @@ internal class RequestBufferingStream : BufferingStream private Stream _innerStream; private Encoding? _encoding; private readonly int _limit; + private bool _hasLogged; public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) : base(logger) @@ -85,10 +86,12 @@ private void WriteToBuffer(ReadOnlySpan span, int res) return; } - if (res == 0) + if (res == 0 && !_hasLogged) { // Done reading, log the string. - LogString(_encoding); + LogString(_encoding, LoggerEventIds.RequestBody, CoreStrings.RequestBody); + _hasLogged = true; + return; } var innerCount = Math.Min(remaining, span.Length); @@ -104,9 +107,10 @@ private void WriteToBuffer(ReadOnlySpan span, int res) BuffersExtensions.Write(this, span.Slice(0, innerCount)); } - if (_limit - _bytesWritten == 0) + if (_limit - _bytesWritten == 0 && !_hasLogged) { - LogString(_encoding); + LogString(_encoding, LoggerEventIds.RequestBody, CoreStrings.RequestBody); + _hasLogged = true; } } From c30c59381afcbbd9db3555cd0345356e6925454e Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 12:41:39 -0700 Subject: [PATCH 19/45] API review feedback --- .../HttpLogging/src/BufferingStream.cs | 1 + .../HttpLogging/src/HttpLoggingFields.cs | 32 ++-- .../HttpLogging/src/HttpLoggingMiddleware.cs | 16 +- .../HttpLogging/src/HttpLoggingOptions.cs | 19 +-- .../HttpLogging/src/MediaTypeOptions.cs | 142 ++++++++++++++++++ .../HttpLogging/src/PublicAPI.Unshipped.txt | 33 ++-- .../test/HttpLoggingMiddlewareTests.cs | 17 +-- 7 files changed, 206 insertions(+), 54 deletions(-) create mode 100644 src/Middleware/HttpLogging/src/MediaTypeOptions.cs diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 36eb4c51a647..17b6703c5b89 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -54,6 +54,7 @@ public void LogString(Encoding? encoding, EventId eventId, string template) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); // If encoding is nullable + // TODO make sure this doesn't truncate. var body = encoding.GetString(ros); _logger.LogInformation(eventId, template, body); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index b9a91937affe..867b92ee2432 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -19,32 +19,32 @@ public enum HttpLoggingFields : long /// /// Request Path /// - Path = 0x1, + RequestPath = 0x1, /// /// Request Query String /// - Query = 0x2, + RequestQuery = 0x2, /// /// Request Protocol /// - Protocol = 0x4, + RequestProtocol = 0x4, /// /// Request Method /// - Method = 0x8, + RequestMethod = 0x8, /// /// Request Scheme /// - Scheme = 0x10, + RequestScheme = 0x10, /// /// Response Status Code /// - StatusCode = 0x20, + ResponseStatusCode = 0x20, /// /// Request Headers @@ -56,20 +56,30 @@ public enum HttpLoggingFields : long /// ResponseHeaders = 0x80, + /// + /// Request Trailers + /// + RequestTrailers = 0x100, + + /// + /// Response Trailers + /// + ResponseTrailers = 0x200, + /// /// Request Body /// - RequestBody = 0x100, + RequestBody = 0x400, /// /// Response Body /// - ResponseBody = 0x200, + ResponseBody = 0x800, /// /// Combination of request properties, including Path, Query, Protocol, Method, and Scheme /// - RequestProperties = Path | Query | Protocol | Method | Scheme, + RequestProperties = RequestPath | RequestQuery | RequestProtocol | RequestMethod | RequestScheme, /// /// Combination of Request Properties and Request Headers @@ -79,7 +89,7 @@ public enum HttpLoggingFields : long /// /// Combination of Response Properties and Response Headers /// - ResponsePropertiesAndHeaders = StatusCode | ResponseHeaders, + ResponsePropertiesAndHeaders = ResponseStatusCode | ResponseHeaders, /// /// Entire Request @@ -89,7 +99,7 @@ public enum HttpLoggingFields : long /// /// Entire Response /// - Response = StatusCode | ResponseHeaders | ResponseBody, + Response = ResponseStatusCode | ResponseHeaders | ResponseBody, /// /// Entire Request and Response diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 3af379986163..e69011d5c7dd 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -68,35 +68,35 @@ public async Task Invoke(HttpContext context) var list = new List>( request.Headers.Count + DefaultRequestFieldsMinusHeaders); - if (options.LoggingFields.HasFlag(HttpLoggingFields.Protocol)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestProtocol)) { AddToList(list, nameof(request.Protocol), request.Protocol); } - if (options.LoggingFields.HasFlag(HttpLoggingFields.Method)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestMethod)) { AddToList(list, nameof(request.Method), request.Method); } - if (options.LoggingFields.HasFlag(HttpLoggingFields.Scheme)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestScheme)) { AddToList(list, nameof(request.Scheme), request.Scheme); } - if (options.LoggingFields.HasFlag(HttpLoggingFields.Path)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestPath)) { AddToList(list, nameof(request.PathBase), request.PathBase); AddToList(list, nameof(request.Path), request.Path); } - if (options.LoggingFields.HasFlag(HttpLoggingFields.Query)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestQuery)) { AddToList(list, nameof(request.QueryString), request.QueryString.Value); } if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders)) { - FilterHeaders(list, request.Headers, options.AllowedRequestHeaders); + FilterHeaders(list, request.Headers, options.RequestHeaders); } if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) @@ -153,14 +153,14 @@ public async Task Invoke(HttpContext context) var list = new List>( response.Headers.Count + DefaultResponseFieldsMinusHeaders); - if (options.LoggingFields.HasFlag(HttpLoggingFields.StatusCode)) + if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode)) { list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); } if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) { - FilterHeaders(list, response.Headers, options.AllowedResponseHeaders); + FilterHeaders(list, response.Headers, options.ResponseHeaders); } var httpResponseLog = new HttpResponseLog(list); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index 8b3c8f0ddabc..fb243572dc06 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -22,10 +22,10 @@ public sealed class HttpLoggingOptions /// Request header values that are allowed to be logged. /// /// - /// If a request header is not present in the , + /// If a request header is not present in the , /// the header name will be logged with a redacted value. /// - public HashSet AllowedRequestHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) + public ISet RequestHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.Accept, HeaderNames.AcceptEncoding, @@ -42,10 +42,10 @@ public sealed class HttpLoggingOptions /// Response header values that are allowed to be logged. /// /// - /// If a response header is not present in the , + /// If a response header is not present in the , /// the header name will be logged with a redacted value. /// - public HashSet AllowedResponseHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) + public ISet ResponseHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.ContentLength, HeaderNames.ContentType, @@ -53,19 +53,12 @@ public sealed class HttpLoggingOptions }; /// - /// A list of supported media type values for request and response body logging with corresponding encodings. + /// Options for configuring encodings for a specific media type. /// /// /// If the request or response do not match the supported media type, the response body will not be logged. /// - public List> SupportedMediaTypes { get; } = new List>() - { - new KeyValuePair(new MediaTypeHeaderValue("application/json"), Encoding.UTF8), - new KeyValuePair(new MediaTypeHeaderValue("application/*+json"), Encoding.UTF8), - new KeyValuePair(new MediaTypeHeaderValue("application/xml"), Encoding.UTF8), - new KeyValuePair(new MediaTypeHeaderValue("application/*+xml"), Encoding.UTF8), - new KeyValuePair(new MediaTypeHeaderValue("text/*"), Encoding.UTF8), - }; + public MediaTypeOptions MediaTypeOptions { get; } = MediaTypeOptions.Default; /// /// Maximum request body size to log (in bytes). Defaults to 32 KB. diff --git a/src/Middleware/HttpLogging/src/MediaTypeOptions.cs b/src/Middleware/HttpLogging/src/MediaTypeOptions.cs new file mode 100644 index 000000000000..8df7d9c8e294 --- /dev/null +++ b/src/Middleware/HttpLogging/src/MediaTypeOptions.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Net.Http.Headers; + +namespace Microsoft.AspNetCore.HttpLogging +{ + /// + /// Options for HttpLogging to configure which encoding to use for each media type. + /// + public class MediaTypeOptions + { + /// + /// Default MediaTypes. Defaults to UTF8 for application/json, application/*+json, + /// application/xml, application/*+xml, and text/*. + /// + public static MediaTypeOptions Default = BuildDefaultMediaTypeOptions(); + + private List _mediaTypeStates = new List(); + + internal MediaTypeOptions() + { + } + + internal List MediaTypeStates => _mediaTypeStates; + + internal static MediaTypeOptions BuildDefaultMediaTypeOptions() + { + var options = new MediaTypeOptions(); + options.AddText("application/json", Encoding.UTF8); + options.AddText("application/*+json", Encoding.UTF8); + options.AddText("application/xml", Encoding.UTF8); + options.AddText("application/*+xml", Encoding.UTF8); + options.AddText("text/*", Encoding.UTF8); + + return options; + } + + /// + /// Adds a to be used for logging as text. + /// + /// The MediaType to add. + public void AddText(MediaTypeHeaderValue mediaType) + { + if (mediaType == null) + { + throw new ArgumentNullException(nameof(mediaType)); + } + + _mediaTypeStates.Add(new MediaTypeState(mediaType) { Encoding = mediaType.Encoding ?? Encoding.UTF8 }); + } + + /// + /// Adds a contentType to be used for logging as text. + /// + /// + /// If charset is not specified in the contentType, the encoding will default to UTF-8. + /// + /// The content type to add. + public void AddText(string contentType) + { + if (contentType == null) + { + throw new ArgumentNullException(nameof(contentType)); + } + + AddText(new MediaTypeHeaderValue(contentType)); + } + + /// + /// Adds a contentType to be used for logging as text. + /// + /// The content type to add. + /// The encoding to use. + public void AddText(string contentType, Encoding encoding) + { + if (contentType == null) + { + throw new ArgumentNullException(nameof(contentType)); + } + + if (encoding == null) + { + throw new ArgumentNullException(nameof(encoding)); + } + + var mediaType = new MediaTypeHeaderValue(contentType); + mediaType.Encoding = encoding; + AddText(mediaType); + } + + /// + /// Adds a to be used for logging as binary. + /// + /// The MediaType to add. + public void AddBinary(MediaTypeHeaderValue mediaType) + { + if (mediaType == null) + { + throw new ArgumentNullException(nameof(mediaType)); + } + + _mediaTypeStates.Add(new MediaTypeState(mediaType) { IsBinary = true }); + } + + /// + /// Adds a content to be used for logging as text. + /// + /// The content type to add. + public void AddBinary(string contentType) + { + if (contentType == null) + { + throw new ArgumentNullException(nameof(contentType)); + } + + AddBinary(new MediaTypeHeaderValue(contentType)); + } + + /// + /// Clears all MediaTypes. + /// + public void Clear() + { + _mediaTypeStates.Clear(); + } + + internal class MediaTypeState + { + public MediaTypeState(MediaTypeHeaderValue mediaTypeHeaderValue) + { + MediaTypeHeaderValue = mediaTypeHeaderValue; + } + + public MediaTypeHeaderValue MediaTypeHeaderValue { get; } + public Encoding? Encoding { get; set; } + public bool IsBinary { get; set; } + } + } +} diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index bccef00ced59..52aada1e2431 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -2,33 +2,42 @@ Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.All = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Method = 8 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.None = 0 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Path = 1 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Protocol = 4 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Query = 2 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Request = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPropertiesAndHeaders | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestBody -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestBody = 256 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestBody = 1024 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestHeaders = 64 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Path | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Query | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Protocol | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Method | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Scheme -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestMethod = 8 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPath = 1 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPath | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestQuery | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProtocol | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestMethod | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestScheme -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestPropertiesAndHeaders = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProperties | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestHeaders -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestProtocol = 4 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestQuery = 2 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestScheme = 16 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.RequestTrailers = 256 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Response = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseBody -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseBody = 512 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseBody = 2048 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders = 128 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.Scheme = 16 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields -Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.StatusCode = 32 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponsePropertiesAndHeaders = Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseStatusCode | Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseHeaders -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseStatusCode = 32 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields +Microsoft.AspNetCore.HttpLogging.HttpLoggingFields.ResponseTrailers = 512 -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedRequestHeaders.get -> System.Collections.Generic.HashSet! -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.AllowedResponseHeaders.get -> System.Collections.Generic.HashSet! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.HttpLoggingOptions() -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.get -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestHeaders.get -> System.Collections.Generic.ISet! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseHeaders.get -> System.Collections.Generic.ISet! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List>! +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddBinary(Microsoft.Net.Http.Headers.MediaTypeHeaderValue! mediaType) -> void +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddBinary(string! contentType) -> void +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(Microsoft.Net.Http.Headers.MediaTypeHeaderValue! mediaType) -> void +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType) -> void +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType, System.Text.Encoding! encoding) -> void Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! +static Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.Default -> Microsoft.AspNetCore.HttpLogging.MediaTypeOptions! static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 1d1c16238d01..afc726add437 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -260,8 +260,8 @@ public async Task UnknownRequestHeadersRedacted() public async Task CanConfigureRequestAllowList() { var options = CreateOptionsAccessor(); - options.CurrentValue.AllowedRequestHeaders.Clear(); - options.CurrentValue.AllowedRequestHeaders.Add("foo"); + options.CurrentValue.RequestHeaders.Clear(); + options.CurrentValue.RequestHeaders.Add("foo"); var middleware = new HttpLoggingMiddleware( c => { @@ -470,11 +470,8 @@ public async Task DifferentEncodingsWork() var expected = new string('a', 1000); var options = CreateOptionsAccessor(); options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; - options.CurrentValue.SupportedMediaTypes.Clear(); - options.CurrentValue.SupportedMediaTypes.Add( - new KeyValuePair( - new MediaTypeHeaderValue("text/plain"), - encoding)); + options.CurrentValue.MediaTypeOptions.Clear(); + options.CurrentValue.MediaTypeOptions.AddText("text/plain", encoding); var middleware = new HttpLoggingMiddleware( async c => @@ -557,7 +554,7 @@ public async Task ResponseInfoLogsAll() public async Task StatusCodeLogs() { var options = CreateOptionsAccessor(); - options.CurrentValue.LoggingFields = HttpLoggingFields.StatusCode; + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseStatusCode; var middleware = new HttpLoggingMiddleware( async c => @@ -629,8 +626,8 @@ public async Task AllowedResponseHeadersModify() { var options = CreateOptionsAccessor(); options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseHeaders; - options.CurrentValue.AllowedResponseHeaders.Clear(); - options.CurrentValue.AllowedResponseHeaders.Add("Test"); + options.CurrentValue.ResponseHeaders.Clear(); + options.CurrentValue.ResponseHeaders.Add("Test"); var middleware = new HttpLoggingMiddleware( c => From 050f7c0d408337e9b9633b60b96b5f75a9f750bd Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 14:49:56 -0700 Subject: [PATCH 20/45] Tests working and most of the feedback --- .../HttpLogging/src/BufferingStream.cs | 103 ++++++---- .../HttpLogging/src/CoreStrings.resx | 126 ------------ .../HttpLogging/src/HttpLoggingExtensions.cs | 11 +- .../HttpLogging/src/HttpLoggingMiddleware.cs | 30 +-- .../HttpLogging/src/HttpLoggingOptions.cs | 2 +- .../HttpLogging/src/MediaTypeHelpers.cs | 20 +- .../HttpLogging/src/MediaTypeOptions.cs | 22 +-- .../HttpLogging/src/PublicAPI.Unshipped.txt | 4 +- .../HttpLogging/src/RequestBufferingStream.cs | 89 ++------- .../src/ResponseBufferingStream.cs | 55 +----- .../test/HttpLoggingMiddlewareTests.cs | 187 +++++++----------- 11 files changed, 199 insertions(+), 450 deletions(-) delete mode 100644 src/Middleware/HttpLogging/src/CoreStrings.resx diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 17b6703c5b89..1af863df2b50 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -3,62 +3,81 @@ using System; using System.Buffers; -using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.IO.Pipelines; -using System.Linq; using System.Runtime.CompilerServices; using System.Text; +using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.HttpLogging { internal class BufferingStream : Stream, IBufferWriter { private const int MinimumBufferSize = 4096; // 4K - private readonly ILogger _logger; protected int _bytesWritten; private BufferSegment? _head; private BufferSegment? _tail; protected Memory _tailMemory; // remainder of tail memory protected int _tailBytesBuffered; - public BufferingStream(ILogger logger) + protected Stream _innerStream; + + public BufferingStream(Stream innerStream) { - _logger = logger; + _innerStream = innerStream; } - public override bool CanRead => throw new NotImplementedException(); + public override bool CanSeek => _innerStream.CanSeek; + + public override bool CanRead => _innerStream.CanRead; - public override bool CanSeek => throw new NotImplementedException(); + public override bool CanWrite => _innerStream.CanWrite; - public override bool CanWrite => throw new NotImplementedException(); + public override long Length => _innerStream.Length; - public override long Length => throw new NotImplementedException(); + public override long Position + { + get => _innerStream.Position; + set => _innerStream.Position = value; + } - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public override int WriteTimeout + { + get => _innerStream.WriteTimeout; + set => _innerStream.WriteTimeout = value; + } - public void LogString(Encoding? encoding, EventId eventId, string template) + public string GetString(Encoding? encoding) { - if (_head == null || _tail == null || encoding == null) + try { - return; - } + if (_head == null || _tail == null || encoding == null) + { + return ""; + } - // Only place where we are actually using the buffered data. - // update tail here. - _tail.End = _tailBytesBuffered; + // Only place where we are actually using the buffered data. + // update tail here. + _tail.End = _tailBytesBuffered; - var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); + var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); - // If encoding is nullable - // TODO make sure this doesn't truncate. - var body = encoding.GetString(ros); - _logger.LogInformation(eventId, template, body); + // TODO make sure this doesn't truncate. + var body = encoding.GetString(ros); - Reset(); + return body; + } + catch (DecoderFallbackException) + { + // TODO log + return ""; + } + finally + { + Reset(); + } } public void Advance(int bytes) @@ -75,21 +94,21 @@ public void Advance(int bytes) public Memory GetMemory(int sizeHint = 0) { - AllocateMemoryUnsynchronized(sizeHint); + AllocateMemory(sizeHint); return _tailMemory; } public Span GetSpan(int sizeHint = 0) { - AllocateMemoryUnsynchronized(sizeHint); + AllocateMemory(sizeHint); return _tailMemory.Span; } - private void AllocateMemoryUnsynchronized(int sizeHint) + private void AllocateMemory(int sizeHint) { if (_head == null) { // We need to allocate memory to write since nobody has written before - BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + BufferSegment newSegment = AllocateSegment(sizeHint); // Set all the pointers _head = _tail = newSegment; @@ -110,7 +129,7 @@ private void AllocateMemoryUnsynchronized(int sizeHint) _tailBytesBuffered = 0; } - BufferSegment newSegment = AllocateSegmentUnsynchronized(sizeHint); + BufferSegment newSegment = AllocateSegment(sizeHint); _tail.SetNext(newSegment); _tail = newSegment; @@ -118,9 +137,9 @@ private void AllocateMemoryUnsynchronized(int sizeHint) } } - private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) + private BufferSegment AllocateSegment(int sizeHint) { - BufferSegment newSegment = CreateSegmentUnsynchronized(); + BufferSegment newSegment = CreateSegment(); // We can't use the recommended pool so use the ArrayPool newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); @@ -130,7 +149,7 @@ private BufferSegment AllocateSegmentUnsynchronized(int sizeHint) return newSegment; } - private BufferSegment CreateSegmentUnsynchronized() + private BufferSegment CreateSegment() { return new BufferSegment(); } @@ -177,27 +196,37 @@ public void Reset() public override void Flush() { - throw new NotImplementedException(); + _innerStream.Flush(); } public override int Read(byte[] buffer, int offset, int count) { - throw new NotImplementedException(); + return _innerStream.Read(buffer, offset, count); + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return _innerStream.ReadAsync(buffer, offset, count, cancellationToken); } public override long Seek(long offset, SeekOrigin origin) { - throw new NotImplementedException(); + return _innerStream.Seek(offset, origin); } public override void SetLength(long value) { - throw new NotImplementedException(); + _innerStream.SetLength(value); } public override void Write(byte[] buffer, int offset, int count) { - throw new NotImplementedException(); + _innerStream.Write(buffer, offset, count); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return _innerStream.WriteAsync(buffer, offset, count, cancellationToken); } } } diff --git a/src/Middleware/HttpLogging/src/CoreStrings.resx b/src/Middleware/HttpLogging/src/CoreStrings.resx deleted file mode 100644 index 3f677a3bf28d..000000000000 --- a/src/Middleware/HttpLogging/src/CoreStrings.resx +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - RequestBody: {0} - - - ResponseBody: {0} - - diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs index 6931964a6496..0300de368d31 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -8,8 +8,13 @@ namespace Microsoft.AspNetCore.HttpLogging { internal static class HttpLoggingExtensions { - static HttpLoggingExtensions() - { - } + private static readonly Action _requestBody = + LoggerMessage.Define(LogLevel.Information, LoggerEventIds.RequestBody, "RequestBody: {Body}"); + + private static readonly Action _responseBody = + LoggerMessage.Define(LogLevel.Information, LoggerEventIds.ResponseBody, "ResponseBody: {Body}"); + + public static void RequestBody(this ILogger logger, string body) => _requestBody(logger, body, null); + public static void ResponseBody(this ILogger logger, string body) => _responseBody(logger, body, null); } } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index e69011d5c7dd..92ebd03d8f59 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -27,14 +27,15 @@ internal sealed class HttpLoggingMiddleware private IOptionsMonitor _options; private const int DefaultRequestFieldsMinusHeaders = 7; private const int DefaultResponseFieldsMinusHeaders = 2; + private const string Redacted = "X"; /// /// Initializes . /// /// /// - /// - public HttpLoggingMiddleware(RequestDelegate next, IOptionsMonitor options, ILoggerFactory loggerFactory) + /// + public HttpLoggingMiddleware(RequestDelegate next, IOptionsMonitor options, ILogger logger) { _next = next ?? throw new ArgumentNullException(nameof(next)); @@ -43,13 +44,13 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptionsMonitor(); + _logger = logger; } /// @@ -62,6 +63,7 @@ public async Task Invoke(HttpContext context) var options = _options.CurrentValue; RequestBufferingStream? requestBufferingStream = null; Stream? originalBody = null; + if ((HttpLoggingFields.Request & options.LoggingFields) != HttpLoggingFields.None) { var request = context.Request; @@ -101,7 +103,7 @@ public async Task Invoke(HttpContext context) if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) && MediaTypeHelpers.TryGetEncodingForMediaType(request.ContentType, - options.SupportedMediaTypes, + options.MediaTypeOptions.MediaTypeStates, out var encoding)) { originalBody = request.Body; @@ -113,7 +115,6 @@ public async Task Invoke(HttpContext context) request.Body = requestBufferingStream; } - // TODO add and remove things from log. var httpRequestLog = new HttpRequestLog(list); _logger.Log(LogLevel.Information, @@ -131,7 +132,7 @@ public async Task Invoke(HttpContext context) if ((HttpLoggingFields.Response & options.LoggingFields) == HttpLoggingFields.None) { // Short circuit and don't replace response body. - await _next(context).ConfigureAwait(false); + await _next(context); return; } @@ -143,13 +144,12 @@ public async Task Invoke(HttpContext context) // TODO pool these. responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, options.ResponseBodyLogLimit, - _logger, context, - options.SupportedMediaTypes); + options.MediaTypeOptions.MediaTypeStates); response.Body = responseBufferingStream; } - await _next(context).ConfigureAwait(false); + await _next(context); var list = new List>( response.Headers.Count + DefaultResponseFieldsMinusHeaders); @@ -173,7 +173,11 @@ public async Task Invoke(HttpContext context) if (responseBufferingStream != null) { - responseBufferingStream.LogString(responseBufferingStream.Encoding, LoggerEventIds.ResponseBody, CoreStrings.ResponseBody); + var responseBody = responseBufferingStream.GetString(responseBufferingStream.Encoding); + if (responseBody != null) + { + _logger.ResponseBody(responseBody); + } } } finally @@ -208,7 +212,7 @@ private void FilterHeaders(List> keyValues, if (!allowedHeaders.Contains(key)) { // Key is not among the "only listed" headers. - keyValues.Add(new KeyValuePair(key, "X")); + keyValues.Add(new KeyValuePair(key, Redacted)); continue; } keyValues.Add(new KeyValuePair(key, value.ToString())); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index fb243572dc06..f5051598206b 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -58,7 +58,7 @@ public sealed class HttpLoggingOptions /// /// If the request or response do not match the supported media type, the response body will not be logged. /// - public MediaTypeOptions MediaTypeOptions { get; } = MediaTypeOptions.Default; + public MediaTypeOptions MediaTypeOptions { get; } = MediaTypeOptions.BuildDefaultMediaTypeOptions(); /// /// Maximum request body size to log (in bytes). Defaults to 32 KB. diff --git a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs index 1f5c18281e34..4e8f226148f0 100644 --- a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs +++ b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs @@ -1,15 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; using System.Collections.Generic; -using System.Linq; +using System.Diagnostics.CodeAnalysis; using System.Text; -using System.Threading.Tasks; using Microsoft.Net.Http.Headers; +using static Microsoft.AspNetCore.HttpLogging.MediaTypeOptions; namespace Microsoft.AspNetCore.HttpLogging { internal static class MediaTypeHelpers { - private static List SupportedEncodings = new List() + private static readonly List SupportedEncodings = new List() { Encoding.UTF8, Encoding.Unicode, @@ -17,7 +20,7 @@ internal static class MediaTypeHelpers Encoding.Latin1 // TODO allowed by default? Make this configurable? }; - public static bool TryGetEncodingForMediaType(string contentType, List> mediaTypeList, out Encoding? encoding) + public static bool TryGetEncodingForMediaType(string contentType, List mediaTypeList, [NotNullWhen(true)] out Encoding? encoding) { encoding = null; if (mediaTypeList == null || mediaTypeList.Count == 0 || string.IsNullOrEmpty(contentType)) @@ -48,12 +51,13 @@ public static bool TryGetEncodingForMediaType(string contentType, List /// Options for HttpLogging to configure which encoding to use for each media type. /// - public class MediaTypeOptions + public sealed class MediaTypeOptions { - /// - /// Default MediaTypes. Defaults to UTF8 for application/json, application/*+json, - /// application/xml, application/*+xml, and text/*. - /// - public static MediaTypeOptions Default = BuildDefaultMediaTypeOptions(); - private List _mediaTypeStates = new List(); internal MediaTypeOptions() @@ -97,12 +91,7 @@ public void AddText(string contentType, Encoding encoding) /// The MediaType to add. public void AddBinary(MediaTypeHeaderValue mediaType) { - if (mediaType == null) - { - throw new ArgumentNullException(nameof(mediaType)); - } - - _mediaTypeStates.Add(new MediaTypeState(mediaType) { IsBinary = true }); + throw new NotSupportedException(); } /// @@ -111,12 +100,7 @@ public void AddBinary(MediaTypeHeaderValue mediaType) /// The content type to add. public void AddBinary(string contentType) { - if (contentType == null) - { - throw new ArgumentNullException(nameof(contentType)); - } - - AddBinary(new MediaTypeHeaderValue(contentType)); + throw new NotSupportedException(); } /// diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index 52aada1e2431..986d071efcef 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -24,20 +24,20 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.HttpLoggingOptions() -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.get -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.LoggingFields.set -> void +Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.MediaTypeOptions.get -> Microsoft.AspNetCore.HttpLogging.MediaTypeOptions! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestBodyLogLimit.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.RequestHeaders.get -> System.Collections.Generic.ISet! Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.get -> int Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseBodyLogLimit.set -> void Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseHeaders.get -> System.Collections.Generic.ISet! -Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.SupportedMediaTypes.get -> System.Collections.Generic.List>! Microsoft.AspNetCore.HttpLogging.MediaTypeOptions Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddBinary(Microsoft.Net.Http.Headers.MediaTypeHeaderValue! mediaType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddBinary(string! contentType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(Microsoft.Net.Http.Headers.MediaTypeHeaderValue! mediaType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType, System.Text.Encoding! encoding) -> void +Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.Clear() -> void Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! -static Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.Default -> Microsoft.AspNetCore.HttpLogging.MediaTypeOptions! static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index ccc8721e70d0..d7b27a1178cc 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -14,41 +14,22 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal class RequestBufferingStream : BufferingStream + internal sealed class RequestBufferingStream : BufferingStream { - private Stream _innerStream; private Encoding? _encoding; private readonly int _limit; private bool _hasLogged; + private ILogger _logger; public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) - : base(logger) + : base(innerStream) { + _logger = logger; _limit = limit; _innerStream = innerStream; _encoding = encoding; } - public override bool CanSeek => false; - - public override bool CanRead => true; - - public override bool CanWrite => false; - - public override long Length => throw new NotSupportedException(); - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override int WriteTimeout - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - public override async ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) { var res = await _innerStream.ReadAsync(destination, cancellationToken); @@ -89,7 +70,11 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (res == 0 && !_hasLogged) { // Done reading, log the string. - LogString(_encoding, LoggerEventIds.RequestBody, CoreStrings.RequestBody); + var requestBody = GetString(_encoding); + if (requestBody != null) + { + _logger.RequestBody(requestBody); + } _hasLogged = true; return; } @@ -109,36 +94,15 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (_limit - _bytesWritten == 0 && !_hasLogged) { - LogString(_encoding, LoggerEventIds.RequestBody, CoreStrings.RequestBody); + var requestBody = GetString(_encoding); + if (requestBody != null) + { + _logger.RequestBody(requestBody); + } _hasLogged = true; } } - public override void Write(byte[] buffer, int offset, int count) - => throw new NotSupportedException(); - - public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException(); - } - - public override void SetLength(long value) - { - throw new NotSupportedException(); - } - - public override void Flush() - { - } - - public override Task FlushAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) { return TaskToApm.Begin(ReadAsync(buffer, offset, count), callback, state); @@ -149,30 +113,5 @@ public override int EndRead(IAsyncResult asyncResult) { return TaskToApm.End(asyncResult); } - - /// - public override async Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { - var remaining = _limit - _bytesWritten; - - while (remaining > 0) - { - // reusing inner buffer here between streams - var memory = GetMemory(); - var innerCount = Math.Min(remaining, memory.Length); - - var res = await _innerStream.ReadAsync(memory.Slice(0, innerCount), cancellationToken); - - _tailBytesBuffered += res; - _bytesWritten += res; - _tailMemory = _tailMemory.Slice(res); - - await destination.WriteAsync(memory.Slice(0, res)); - - remaining -= res; - } - - await _innerStream.CopyToAsync(destination, bufferSize, cancellationToken); - } } } diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 743b39f98ed0..e5a782042d4b 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -15,23 +15,21 @@ using Microsoft.AspNetCore.Http.Features; using Microsoft.Extensions.Logging; using Microsoft.Net.Http.Headers; +using static Microsoft.AspNetCore.HttpLogging.MediaTypeOptions; namespace Microsoft.AspNetCore.HttpLogging { /// /// Stream that buffers reads /// - internal class ResponseBufferingStream : BufferingStream, IHttpResponseBodyFeature + internal sealed class ResponseBufferingStream : BufferingStream, IHttpResponseBodyFeature { - private const int MinimumBufferSize = 4096; // 4K - private readonly IHttpResponseBodyFeature _innerBodyFeature; - private readonly Stream _innerStream; private readonly int _limit; private PipeWriter? _pipeAdapter; private readonly HttpContext _context; - private readonly List> _encodings; + private readonly List _encodings; private Encoding? _encoding; private bool _hasCheckedEncoding; @@ -39,12 +37,11 @@ internal class ResponseBufferingStream : BufferingStream, IHttpResponseBodyFeatu private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true); internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, - int limit, ILogger logger, + int limit, HttpContext context, - List> encodings) - : base (logger) + List encodings) + : base(innerBodyFeature.Stream) { - // TODO need first write event _innerBodyFeature = innerBodyFeature; _innerStream = innerBodyFeature.Stream; _limit = limit; @@ -52,21 +49,6 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, _encodings = encodings; } - public override bool CanRead => false; - public override bool CanSeek => false; - public override bool CanWrite => _innerStream.CanWrite; - - public override long Length - { - get { throw new NotSupportedException(); } - } - - public override long Position - { - get { throw new NotSupportedException(); } - set { throw new NotSupportedException(); } - } - public Stream Stream => this; public PipeWriter Writer @@ -84,31 +66,6 @@ public PipeWriter Writer public Encoding? Encoding { get => _encoding; } - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotSupportedException(); - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException(); - } - - public override void SetLength(long value) - { - throw new NotSupportedException(); - } - - public override void Flush() - { - _innerStream.Flush(); - } - - public override Task FlushAsync(CancellationToken cancellationToken) - { - return _innerStream.FlushAsync(cancellationToken); - } - public override void Write(byte[] buffer, int offset, int count) { Write(buffer.AsSpan(offset, count)); diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index afc726add437..7cfe2a24ff70 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -12,6 +12,7 @@ using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.Options; using Microsoft.Net.Http.Headers; +using Microsoft.Extensions.Logging; using Moq; using Xunit; @@ -35,14 +36,17 @@ public static TheoryData BodyData [Fact] public void Ctor_ThrowsExceptionsWhenNullArgs() { - Assert.Throws(() => new HttpLoggingMiddleware(null, CreateOptionsAccessor(), LoggerFactory)); + Assert.Throws(() => new HttpLoggingMiddleware( + null, + CreateOptionsAccessor(), + LoggerFactory.CreateLogger())); Assert.Throws(() => new HttpLoggingMiddleware(c => { return Task.CompletedTask; }, null, - LoggerFactory)); + LoggerFactory.CreateLogger())); Assert.Throws(() => new HttpLoggingMiddleware(c => { @@ -65,7 +69,7 @@ public async Task NoopWhenLoggingDisabled() return Task.CompletedTask; }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.Protocol = "HTTP/1.0"; @@ -95,12 +99,20 @@ public async Task NoopWhenLoggingDisabled() public async Task DefaultRequestInfoOnlyHeadersAndRequestInfo() { var middleware = new HttpLoggingMiddleware( - c => + async c => { - return Task.CompletedTask; + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } }, CreateOptionsAccessor(), - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.Protocol = "HTTP/1.0"; @@ -143,7 +155,7 @@ public async Task RequestLogsAllRequestInfo() } }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.Protocol = "HTTP/1.0"; @@ -173,12 +185,20 @@ public async Task RequestPropertiesLogs() var options = CreateOptionsAccessor(); options.CurrentValue.LoggingFields = HttpLoggingFields.RequestProperties; var middleware = new HttpLoggingMiddleware( - c => + async c => { - return Task.CompletedTask; + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.Protocol = "HTTP/1.0"; @@ -208,12 +228,20 @@ public async Task RequestHeadersLogs() var options = CreateOptionsAccessor(); options.CurrentValue.LoggingFields = HttpLoggingFields.RequestHeaders; var middleware = new HttpLoggingMiddleware( - c => + async c => { - return Task.CompletedTask; + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.Protocol = "HTTP/1.0"; @@ -241,12 +269,20 @@ public async Task RequestHeadersLogs() public async Task UnknownRequestHeadersRedacted() { var middleware = new HttpLoggingMiddleware( - c => + async c => { - return Task.CompletedTask; + var arr = new byte[4096]; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + } }, CreateOptionsAccessor(), - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -254,6 +290,7 @@ public async Task UnknownRequestHeadersRedacted() await middleware.Invoke(httpContext); Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: X")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("foo: bar")); } [Fact] @@ -268,7 +305,7 @@ public async Task CanConfigureRequestAllowList() return Task.CompletedTask; }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -279,7 +316,9 @@ public async Task CanConfigureRequestAllowList() await middleware.Invoke(httpContext); Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: bar")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("foo: X")); Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: X")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); } [Theory] @@ -303,7 +342,7 @@ public async Task RequestBodyReadingWorks(string expected) } }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.ContentType = "text/plain"; @@ -316,39 +355,6 @@ public async Task RequestBodyReadingWorks(string expected) [Fact] public async Task RequestBodyReadingLimitWorks() - { - var input = string.Concat(new string('a', 30000), new string('b', 3000)); - var options = CreateOptionsAccessor(); - options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; - - var middleware = new HttpLoggingMiddleware( - async c => - { - var arr = new byte[4096]; - while (true) - { - var res = await c.Request.Body.ReadAsync(arr); - if (res == 0) - { - break; - } - } - }, - options, - LoggerFactory); - - var httpContext = new DefaultHttpContext(); - httpContext.Request.ContentType = "text/plain"; - httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); - - await middleware.Invoke(httpContext); - var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit); - - Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); - } - - [Fact] - public async Task RequestBodyReadingLimitGreaterThanPipeWorks() { var input = string.Concat(new string('a', 60000), new string('b', 3000)); var options = CreateOptionsAccessor(); @@ -372,7 +378,7 @@ public async Task RequestBodyReadingLimitGreaterThanPipeWorks() Assert.Equal(63000, count); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.ContentType = "text/plain"; @@ -381,7 +387,7 @@ public async Task RequestBodyReadingLimitGreaterThanPipeWorks() await middleware.Invoke(httpContext); var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit); - Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); + Assert.Contains(TestSink.Writes, w => w.Message.Equals("RequestBody: " + expected)); } [Theory] @@ -412,7 +418,7 @@ public async Task VerifyDefaultMediaTypeHeaders(string contentType) } }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.ContentType = contentType; @@ -452,7 +458,7 @@ public async Task RejectedContentTypes(string contentType) Assert.Equal(1000, count); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.ContentType = contentType; @@ -491,7 +497,7 @@ public async Task DifferentEncodingsWork() Assert.Equal(2000, count); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); httpContext.Request.ContentType = "text/plain"; @@ -514,7 +520,7 @@ public async Task DefaultResponseInfoOnlyHeadersAndRequestInfo() await c.Response.WriteAsync("test"); }, CreateOptionsAccessor(), - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -539,7 +545,7 @@ public async Task ResponseInfoLogsAll() await c.Response.WriteAsync("test"); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -565,7 +571,7 @@ public async Task StatusCodeLogs() await c.Response.WriteAsync("test"); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -590,7 +596,7 @@ public async Task ResponseHeadersLogs() await c.Response.WriteAsync("test"); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -613,7 +619,7 @@ public async Task ResponseHeadersRedacted() return Task.CompletedTask; }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -637,7 +643,7 @@ public async Task AllowedResponseHeadersModify() return Task.CompletedTask; }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -659,7 +665,7 @@ public async Task ResponseBodyWritingWorks(string expected) return c.Response.WriteAsync(expected); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -681,7 +687,7 @@ public async Task ResponseBodyWritingLimitWorks() return c.Response.WriteAsync(input); }, options, - LoggerFactory); + LoggerFactory.CreateLogger()); var httpContext = new DefaultHttpContext(); @@ -697,58 +703,5 @@ private IOptionsMonitor CreateOptionsAccessor() var optionsAccessor = Mock.Of>(o => o.CurrentValue == options); return optionsAccessor; } - - private class SlowStream : Stream - { - private readonly Stream _inner; - private readonly TimeSpan _artificialDelay; - - public SlowStream(Stream inner, TimeSpan artificialDelay) - { - _inner = inner; - _artificialDelay = artificialDelay; - } - - public override void Flush() - { - _inner.Flush(); - } - - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken token) - { - await Task.Delay(_artificialDelay); - return await _inner.ReadAsync(buffer, offset, count, token); - } - - public override void SetLength(long value) - { - _inner.SetLength(value); - } - - public override void Write(byte[] buffer, int offset, int count) - { - _inner.Write(buffer, offset, count); - } - - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotImplementedException(); - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotImplementedException(); - } - - public override bool CanRead => _inner.CanRead; - public override bool CanSeek => _inner.CanSeek; - public override bool CanWrite => _inner.CanWrite; - public override long Length => _inner.Length; - public override long Position - { - get => _inner.Position; - set => _inner.Position = value; - } - } } } From 74ddd32554f5fd7a4773a28d1a365345c1900923 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 14:57:29 -0700 Subject: [PATCH 21/45] rename --- src/Middleware/HttpLogging/src/BufferingStream.cs | 6 +++--- src/Middleware/HttpLogging/src/RequestBufferingStream.cs | 6 +++--- src/Middleware/HttpLogging/src/ResponseBufferingStream.cs | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 1af863df2b50..afe6df7e32a1 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.HttpLogging internal class BufferingStream : Stream, IBufferWriter { private const int MinimumBufferSize = 4096; // 4K - protected int _bytesWritten; + protected int _bytesBuffered; private BufferSegment? _head; private BufferSegment? _tail; protected Memory _tailMemory; // remainder of tail memory @@ -88,7 +88,7 @@ public void Advance(int bytes) } _tailBytesBuffered += bytes; - _bytesWritten += bytes; + _bytesBuffered += bytes; _tailMemory = _tailMemory.Slice(bytes); } @@ -185,7 +185,7 @@ public void Reset() _head = _tail = null; - _bytesWritten = 0; + _bytesBuffered = 0; _tailBytesBuffered = 0; } diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index d7b27a1178cc..b9caaf386b62 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -60,7 +60,7 @@ public override int Read(byte[] buffer, int offset, int count) private void WriteToBuffer(ReadOnlySpan span, int res) { // get what was read into the buffer - var remaining = _limit - _bytesWritten; + var remaining = _limit - _bytesBuffered; if (remaining == 0) { @@ -84,7 +84,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) { _tailBytesBuffered += innerCount; - _bytesWritten += innerCount; + _bytesBuffered += innerCount; _tailMemory = _tailMemory.Slice(innerCount); } else @@ -92,7 +92,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) BuffersExtensions.Write(this, span.Slice(0, innerCount)); } - if (_limit - _bytesWritten == 0 && !_hasLogged) + if (_limit - _bytesBuffered == 0 && !_hasLogged) { var requestBody = GetString(_encoding); if (requestBody != null) diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index e5a782042d4b..04a8f8406624 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -83,7 +83,7 @@ public override void EndWrite(IAsyncResult asyncResult) public override void Write(ReadOnlySpan span) { - var remaining = _limit - _bytesWritten; + var remaining = _limit - _bytesBuffered; var innerCount = Math.Min(remaining, span.Length); OnFirstWrite(); @@ -91,7 +91,7 @@ public override void Write(ReadOnlySpan span) if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) { _tailBytesBuffered += innerCount; - _bytesWritten += innerCount; + _bytesBuffered += innerCount; _tailMemory = _tailMemory.Slice(innerCount); } else @@ -104,7 +104,7 @@ public override void Write(ReadOnlySpan span) public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - var remaining = _limit - _bytesWritten; + var remaining = _limit - _bytesBuffered; var innerCount = Math.Min(remaining, count); OnFirstWrite(); @@ -113,7 +113,7 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc { buffer.AsSpan(offset, count).CopyTo(_tailMemory.Span); _tailBytesBuffered += innerCount; - _bytesWritten += innerCount; + _bytesBuffered += innerCount; _tailMemory = _tailMemory.Slice(innerCount); } else From 5d55650925689ac7be391156da25577cd904849f Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 16:04:47 -0700 Subject: [PATCH 22/45] Feedback --- src/Middleware/HttpLogging/src/BufferingStream.cs | 13 +++++++------ .../HttpLogging/src/HttpLoggingExtensions.cs | 4 ++++ .../HttpLogging/src/HttpLoggingMiddleware.cs | 1 + src/Middleware/HttpLogging/src/LoggerEventIds.cs | 1 + .../HttpLogging/src/RequestBufferingStream.cs | 2 +- .../HttpLogging/src/ResponseBufferingStream.cs | 3 ++- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index afe6df7e32a1..32f90e69add4 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.HttpLogging { @@ -21,11 +22,12 @@ internal class BufferingStream : Stream, IBufferWriter private BufferSegment? _tail; protected Memory _tailMemory; // remainder of tail memory protected int _tailBytesBuffered; - + private ILogger _logger; protected Stream _innerStream; - public BufferingStream(Stream innerStream) + public BufferingStream(Stream innerStream, ILogger logger) { + _logger = logger; _innerStream = innerStream; } @@ -64,15 +66,14 @@ public string GetString(Encoding? encoding) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); - // TODO make sure this doesn't truncate. var body = encoding.GetString(ros); return body; } - catch (DecoderFallbackException) + catch (DecoderFallbackException ex) { - // TODO log - return ""; + _logger.DecodeFailure(ex); + return ""; } finally { diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs index 0300de368d31..358aac58a00a 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -14,7 +14,11 @@ internal static class HttpLoggingExtensions private static readonly Action _responseBody = LoggerMessage.Define(LogLevel.Information, LoggerEventIds.ResponseBody, "ResponseBody: {Body}"); + private static readonly Action _decodeFailure = + LoggerMessage.Define(LogLevel.Debug, LoggerEventIds.DecodeFailure, "Decode failure while converting body."); + public static void RequestBody(this ILogger logger, string body) => _requestBody(logger, body, null); public static void ResponseBody(this ILogger logger, string body) => _responseBody(logger, body, null); + public static void DecodeFailure(this ILogger logger, Exception ex) => _decodeFailure(logger, ex); } } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 92ebd03d8f59..6713024a6f2d 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -144,6 +144,7 @@ public async Task Invoke(HttpContext context) // TODO pool these. responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, options.ResponseBodyLogLimit, + _logger, context, options.MediaTypeOptions.MediaTypeStates); response.Body = responseBufferingStream; diff --git a/src/Middleware/HttpLogging/src/LoggerEventIds.cs b/src/Middleware/HttpLogging/src/LoggerEventIds.cs index ddc4f8e5e8c8..df3d441a3108 100644 --- a/src/Middleware/HttpLogging/src/LoggerEventIds.cs +++ b/src/Middleware/HttpLogging/src/LoggerEventIds.cs @@ -11,5 +11,6 @@ internal static class LoggerEventIds public static readonly EventId ResponseLog = new EventId(2, "ResponseLog"); public static readonly EventId RequestBody = new EventId(3, "RequestBody"); public static readonly EventId ResponseBody = new EventId(4, "ResponseBody"); + public static readonly EventId DecodeFailure = new EventId(5, "DecodeFaulure"); } } diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index b9caaf386b62..18cf45b45a8c 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -22,7 +22,7 @@ internal sealed class RequestBufferingStream : BufferingStream private ILogger _logger; public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) - : base(innerStream) + : base(innerStream, logger) { _logger = logger; _limit = limit; diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 04a8f8406624..8540868f3fd0 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -38,9 +38,10 @@ internal sealed class ResponseBufferingStream : BufferingStream, IHttpResponseBo internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit, + ILogger logger, HttpContext context, List encodings) - : base(innerBodyFeature.Stream) + : base(innerBodyFeature.Stream, logger) { _innerBodyFeature = innerBodyFeature; _innerStream = innerBodyFeature.Stream; From 1ad55fb97d010f3bffe45f00432b4a558c131594 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 16:13:05 -0700 Subject: [PATCH 23/45] bit more logging --- src/Middleware/HttpLogging/src/BufferingStream.cs | 9 ++++++++- src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs | 4 ++++ src/Middleware/HttpLogging/src/LoggerEventIds.cs | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 32f90e69add4..f355dd09fef9 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -55,11 +55,18 @@ public string GetString(Encoding? encoding) { try { - if (_head == null || _tail == null || encoding == null) + if (_head == null || _tail == null) { + // nothing written return ""; } + if (encoding == null) + { + _logger.UnrecognizedMediaType(); + return ""; + } + // Only place where we are actually using the buffered data. // update tail here. _tail.End = _tailBytesBuffered; diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs index 358aac58a00a..895a52c7f9ab 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -17,8 +17,12 @@ internal static class HttpLoggingExtensions private static readonly Action _decodeFailure = LoggerMessage.Define(LogLevel.Debug, LoggerEventIds.DecodeFailure, "Decode failure while converting body."); + private static readonly Action _unrecognizedMediaType = + LoggerMessage.Define(LogLevel.Debug, LoggerEventIds.UnrecognizedMediaType, "Unrecognized Content-Type for body."); + public static void RequestBody(this ILogger logger, string body) => _requestBody(logger, body, null); public static void ResponseBody(this ILogger logger, string body) => _responseBody(logger, body, null); public static void DecodeFailure(this ILogger logger, Exception ex) => _decodeFailure(logger, ex); + public static void UnrecognizedMediaType(this ILogger logger) => _unrecognizedMediaType(logger, null); } } diff --git a/src/Middleware/HttpLogging/src/LoggerEventIds.cs b/src/Middleware/HttpLogging/src/LoggerEventIds.cs index df3d441a3108..a1d156953738 100644 --- a/src/Middleware/HttpLogging/src/LoggerEventIds.cs +++ b/src/Middleware/HttpLogging/src/LoggerEventIds.cs @@ -12,5 +12,6 @@ internal static class LoggerEventIds public static readonly EventId RequestBody = new EventId(3, "RequestBody"); public static readonly EventId ResponseBody = new EventId(4, "ResponseBody"); public static readonly EventId DecodeFailure = new EventId(5, "DecodeFaulure"); + public static readonly EventId UnrecognizedMediaType = new EventId(6, "UnrecognizedMediaType"); } } From 67e9222e9946f403d803dcff793990b8d2baf5b4 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 16:45:43 -0700 Subject: [PATCH 24/45] More overloads --- .../HttpLogging/src/BufferingStream.cs | 67 ++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index f355dd09fef9..f469ee2fd92e 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -73,9 +73,22 @@ public string GetString(Encoding? encoding) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); - var body = encoding.GetString(ros); + var bufferWriter = new ArrayBufferWriter(); - return body; + var decoder = encoding.GetDecoder(); + EncodingExtensions.Convert(decoder, ros, bufferWriter, flush: false, out var charUsed, out var completed); + + while (true) + { + if (completed) + { + return new string(bufferWriter.WrittenSpan); + } + else + { + EncodingExtensions.Convert(decoder, ReadOnlySequence.Empty, bufferWriter, flush: true, out charUsed, out completed); + } + } } catch (DecoderFallbackException ex) { @@ -207,6 +220,11 @@ public override void Flush() _innerStream.Flush(); } + public override Task FlushAsync(CancellationToken cancellationToken) + { + return _innerStream.FlushAsync(cancellationToken); + } + public override int Read(byte[] buffer, int offset, int count) { return _innerStream.Read(buffer, offset, count); @@ -236,5 +254,50 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati { return _innerStream.WriteAsync(buffer, offset, count, cancellationToken); } + + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) + { + return _innerStream.BeginRead(buffer, offset, count, callback, state); + } + + public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) + { + return _innerStream.BeginWrite(buffer, offset, count, callback, state); + } + + public override int EndRead(IAsyncResult asyncResult) + { + return _innerStream.EndRead(asyncResult); + } + + public override void EndWrite(IAsyncResult asyncResult) + { + _innerStream.EndWrite(asyncResult); + } + + public override void CopyTo(Stream destination, int bufferSize) + { + _innerStream.CopyTo(destination, bufferSize); + } + + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + return _innerStream.CopyToAsync(destination, bufferSize, cancellationToken); + } + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + return _innerStream.ReadAsync(buffer, cancellationToken); + } + + public override ValueTask DisposeAsync() + { + return _innerStream.DisposeAsync(); + } + + public override int Read(Span buffer) + { + return _innerStream.Read(buffer); + } } } From 34977932839d66ab10c12fa73f6b5f899ec1a545 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 16 Apr 2021 16:49:11 -0700 Subject: [PATCH 25/45] Fixing truncation --- .../test/HttpLoggingMiddlewareTests.cs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 7cfe2a24ff70..dd6c4fa33bf0 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -353,6 +353,43 @@ public async Task RequestBodyReadingWorks(string expected) Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); } + [Fact] + public async Task RequestBodyReadingLimitLongCharactersWorks() + { + var input = string.Concat(new string('あ', 60000)); + var options = CreateOptionsAccessor(); + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var count = 0; + while (true) + { + var res = await c.Request.Body.ReadAsync(arr); + if (res == 0) + { + break; + } + count += res; + } + + Assert.Equal(180000, count); + }, + options, + LoggerFactory.CreateLogger()); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); + + await middleware.Invoke(httpContext); + var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit / 3); + + Assert.Contains(TestSink.Writes, w => w.Message.Equals("RequestBody: " + expected)); + } + [Fact] public async Task RequestBodyReadingLimitWorks() { From 497c91b5892857e27d3827b56930bcc2a7d29b3c Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 08:02:06 -0700 Subject: [PATCH 26/45] Update src/Middleware/HttpLogging/src/HttpRequestLog.cs Co-authored-by: Kahbazi --- src/Middleware/HttpLogging/src/HttpRequestLog.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs index 416f814cd3d9..84c7234e554d 100644 --- a/src/Middleware/HttpLogging/src/HttpRequestLog.cs +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -26,7 +26,8 @@ public HttpRequestLog(List> keyValues) public IEnumerator> GetEnumerator() { - for (var i = 0; i < Count; i++) + var count = _keyValues.Count; + for (var i = 0; i < count; i++) { yield return _keyValues[i]; } From 6231ce3cc9b04c8e1d0cb2fa568a02a84b957697 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 10:03:30 -0700 Subject: [PATCH 27/45] More tests and log headers later --- .../HttpLogging/src/BufferingStream.cs | 2 +- .../HttpLogging/src/HttpLoggingMiddleware.cs | 50 +++++++----- .../HttpLogging/src/MediaTypeHelpers.cs | 3 +- .../HttpLogging/src/MediaTypeOptions.cs | 9 ++- .../HttpLogging/src/RequestBufferingStream.cs | 1 - .../src/ResponseBufferingStream.cs | 30 +++++-- .../test/HttpLoggingDefaultBuilderTests.cs | 12 --- .../test/HttpLoggingMiddlewareTests.cs | 78 ++++++++++++++++++- .../test/HttpLoggingOptionsTests.cs | 67 ++++++++++++++++ 9 files changed, 208 insertions(+), 44 deletions(-) delete mode 100644 src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs create mode 100644 src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index f469ee2fd92e..c09b2e689d5d 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -22,7 +22,7 @@ internal class BufferingStream : Stream, IBufferWriter private BufferSegment? _tail; protected Memory _tailMemory; // remainder of tail memory protected int _tailBytesBuffered; - private ILogger _logger; + protected ILogger _logger; protected Stream _innerStream; public BufferingStream(Stream innerStream, ILogger logger) diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 6713024a6f2d..3645d4394103 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -146,32 +146,20 @@ public async Task Invoke(HttpContext context) options.ResponseBodyLogLimit, _logger, context, - options.MediaTypeOptions.MediaTypeStates); + options.MediaTypeOptions.MediaTypeStates, + options); response.Body = responseBufferingStream; + context.Features.Set(responseBufferingStream); } await _next(context); - var list = new List>( - response.Headers.Count + DefaultResponseFieldsMinusHeaders); - - if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode)) - { - list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); - } - if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) + if (responseBufferingStream == null || responseBufferingStream.FirstWrite == false) { - FilterHeaders(list, response.Headers, options.ResponseHeaders); + // No body, write headers here. + LogResponseHeaders(response, options, _logger); } - var httpResponseLog = new HttpResponseLog(list); - - _logger.Log(LogLevel.Information, - eventId: LoggerEventIds.ResponseLog, - state: httpResponseLog, - exception: null, - formatter: HttpResponseLog.Callback); - if (responseBufferingStream != null) { var responseBody = responseBufferingStream.GetString(responseBufferingStream.Encoding); @@ -204,7 +192,31 @@ private static void AddToList(List> list, string k list.Add(new KeyValuePair(key, value)); } - private void FilterHeaders(List> keyValues, + public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions options, ILogger logger) + { + var list = new List>( + response.Headers.Count + DefaultResponseFieldsMinusHeaders); + + if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode)) + { + list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); + } + + if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) + { + FilterHeaders(list, response.Headers, options.ResponseHeaders); + } + + var httpResponseLog = new HttpResponseLog(list); + + logger.Log(LogLevel.Information, + eventId: LoggerEventIds.ResponseLog, + state: httpResponseLog, + exception: null, + formatter: HttpResponseLog.Callback); + } + + internal static void FilterHeaders(List> keyValues, IHeaderDictionary headers, ISet allowedHeaders) { diff --git a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs index 4e8f226148f0..f38e4ec551d6 100644 --- a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs +++ b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs @@ -57,7 +57,8 @@ public static bool TryGetEncodingForMediaType(string contentType, List @@ -60,7 +62,7 @@ public void AddText(string contentType) throw new ArgumentNullException(nameof(contentType)); } - AddText(new MediaTypeHeaderValue(contentType)); + AddText(MediaTypeHeaderValue.Parse(contentType)); } /// @@ -80,7 +82,7 @@ public void AddText(string contentType, Encoding encoding) throw new ArgumentNullException(nameof(encoding)); } - var mediaType = new MediaTypeHeaderValue(contentType); + var mediaType = MediaTypeHeaderValue.Parse(contentType); mediaType.Encoding = encoding; AddText(mediaType); } @@ -119,7 +121,6 @@ public MediaTypeState(MediaTypeHeaderValue mediaTypeHeaderValue) } public MediaTypeHeaderValue MediaTypeHeaderValue { get; } - public Encoding? Encoding { get; set; } public bool IsBinary { get; set; } } } diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index 18cf45b45a8c..ef69dd6f358a 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -19,7 +19,6 @@ internal sealed class RequestBufferingStream : BufferingStream private Encoding? _encoding; private readonly int _limit; private bool _hasLogged; - private ILogger _logger; public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) : base(innerStream, logger) diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index 8540868f3fd0..ec0fbc002eb4 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -30,9 +30,8 @@ internal sealed class ResponseBufferingStream : BufferingStream, IHttpResponseBo private readonly HttpContext _context; private readonly List _encodings; - + private readonly HttpLoggingOptions _options; private Encoding? _encoding; - private bool _hasCheckedEncoding; private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true); @@ -40,7 +39,8 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, int limit, ILogger logger, HttpContext context, - List encodings) + List encodings, + HttpLoggingOptions options) : base(innerBodyFeature.Stream, logger) { _innerBodyFeature = innerBodyFeature; @@ -48,8 +48,11 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, _limit = limit; _context = context; _encodings = encodings; + _options = options; } + public bool FirstWrite { get; private set; } + public Stream Stream => this; public PipeWriter Writer @@ -127,10 +130,13 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc private void OnFirstWrite() { - if (!_hasCheckedEncoding) + if (!FirstWrite) { + // Log headers as first write occurs (headers locked now) + HttpLoggingMiddleware.LogResponseHeaders(_context.Response, _options, _logger); + MediaTypeHelpers.TryGetEncodingForMediaType(_context.Response.ContentType, _encodings, out _encoding); - _hasCheckedEncoding = true; + FirstWrite = true; } } @@ -141,11 +147,13 @@ public void DisableBuffering() public Task SendFileAsync(string path, long offset, long? count, CancellationToken cancellation) { + OnFirstWrite(); return _innerBodyFeature.SendFileAsync(path, offset, count, cancellation); } public Task StartAsync(CancellationToken token = default) { + OnFirstWrite(); return _innerBodyFeature.StartAsync(token); } @@ -153,5 +161,17 @@ public async Task CompleteAsync() { await _innerBodyFeature.CompleteAsync(); } + + public override void Flush() + { + OnFirstWrite(); + base.Flush(); + } + + public override Task FlushAsync(CancellationToken cancellationToken) + { + OnFirstWrite(); + return base.FlushAsync(cancellationToken); + } } } diff --git a/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs deleted file mode 100644 index f486c7ddc5ed..000000000000 --- a/src/Middleware/HttpLogging/test/HttpLoggingDefaultBuilderTests.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.AspNetCore.HttpLogging.Tests -{ - class HttpLoggingDefaultBuilderTests - { - } -} diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index dd6c4fa33bf0..bd267106ef7f 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -16,7 +16,7 @@ using Moq; using Xunit; -namespace Microsoft.AspNetCore.HttpsPolicy.Tests +namespace Microsoft.AspNetCore.HttpLogging.Tests { public class HttpLoggingMiddlewareTests : LoggedTest { @@ -734,6 +734,82 @@ public async Task ResponseBodyWritingLimitWorks() Assert.Contains(TestSink.Writes, w => w.Message.Contains(expected)); } + [Fact] + public async Task FirstWriteResponseHeadersLogged() + { + var options = CreateOptionsAccessor(); + options.CurrentValue.LoggingFields = HttpLoggingFields.Response; + + var writtenHeaders = new TaskCompletionSource(); + var letBodyFinish = new TaskCompletionSource(); + + var middleware = new HttpLoggingMiddleware( + async c => + { + c.Response.StatusCode = 200; + c.Response.Headers[HeaderNames.TransferEncoding] = "test"; + c.Response.ContentType = "text/plain"; + await c.Response.WriteAsync("test"); + writtenHeaders.SetResult(null); + await letBodyFinish.Task; + }, + options, + LoggerFactory.CreateLogger()); + + var httpContext = new DefaultHttpContext(); + + var middlewareTask = middleware.Invoke(httpContext); + + await writtenHeaders.Task; + + Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Transfer-Encoding: test")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + + letBodyFinish.SetResult(null); + + await middlewareTask; + + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Body: test")); + } + + [Fact] + public async Task StartAsyncResponseHeadersLogged() + { + var options = CreateOptionsAccessor(); + options.CurrentValue.LoggingFields = HttpLoggingFields.Response; + + var writtenHeaders = new TaskCompletionSource(); + var letBodyFinish = new TaskCompletionSource(); + + var middleware = new HttpLoggingMiddleware( + async c => + { + c.Response.StatusCode = 200; + c.Response.Headers[HeaderNames.TransferEncoding] = "test"; + c.Response.ContentType = "text/plain"; + await c.Response.StartAsync(); + writtenHeaders.SetResult(null); + await letBodyFinish.Task; + }, + options, + LoggerFactory.CreateLogger()); + + var httpContext = new DefaultHttpContext(); + + var middlewareTask = middleware.Invoke(httpContext); + + await writtenHeaders.Task; + + Assert.Contains(TestSink.Writes, w => w.Message.Contains("StatusCode: 200")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Transfer-Encoding: test")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Body: test")); + + letBodyFinish.SetResult(null); + + await middlewareTask; + } + private IOptionsMonitor CreateOptionsAccessor() { var options = new HttpLoggingOptions(); diff --git a/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs new file mode 100644 index 000000000000..e657516fe6f5 --- /dev/null +++ b/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace Microsoft.AspNetCore.HttpLogging.Tests +{ + public class HttpLoggingOptionsTests + { + [Fact] + public void DefaultsMediaTypes() + { + var options = new HttpLoggingOptions(); + var defaultMediaTypes = options.MediaTypeOptions.MediaTypeStates; + Assert.Equal(5, defaultMediaTypes.Count); + + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.MediaType.Equals("application/json")); + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.MediaType.Equals("application/*+json")); + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.MediaType.Equals("application/xml")); + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.MediaType.Equals("application/*+xml")); + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.MediaType.Equals("text/*")); + } + + [Fact] + public void CanAddMediaTypesString() + { + var options = new HttpLoggingOptions(); + options.MediaTypeOptions.AddText("test/*"); + + var defaultMediaTypes = options.MediaTypeOptions.MediaTypeStates; + Assert.Equal(6, defaultMediaTypes.Count); + + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.MediaType.Equals("test/*")); + } + + [Fact] + public void CanAddMediaTypesWithCharset() + { + var options = new HttpLoggingOptions(); + options.MediaTypeOptions.AddText("test/*; charset=ascii"); + + var defaultMediaTypes = options.MediaTypeOptions.MediaTypeStates; + Assert.Equal(6, defaultMediaTypes.Count); + + Assert.Contains(defaultMediaTypes, w => w.MediaTypeHeaderValue.Encoding.WebName.Equals("us-ascii")); + } + + [Fact] + public void CanClearMediaTypes() + { + var options = new HttpLoggingOptions(); + options.MediaTypeOptions.Clear(); + Assert.Empty(options.MediaTypeOptions.MediaTypeStates); + } + + [Fact] + public void HeadersAreCaseInsensitive() + { + var options = new HttpLoggingOptions(); + options.RequestHeaders.Clear(); + options.RequestHeaders.Add("Test"); + options.RequestHeaders.Add("test"); + + Assert.Single(options.RequestHeaders); + } + } +} From a1364581f97f8a40e9baa0bbc9df6b99a8ed1fb2 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 10:10:34 -0700 Subject: [PATCH 28/45] Test for invalid media type --- .../test/HttpLoggingMiddlewareTests.cs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index bd267106ef7f..8b09df4c8ec6 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -810,6 +810,28 @@ public async Task StartAsyncResponseHeadersLogged() await middlewareTask; } + [Fact] + public async Task UnrecognizedMediaType() + { + var expected = "Hello world"; + var options = CreateOptionsAccessor(); + options.CurrentValue.LoggingFields = HttpLoggingFields.ResponseBody; + var middleware = new HttpLoggingMiddleware( + c => + { + c.Response.ContentType = "foo/*"; + return c.Response.WriteAsync(expected); + }, + options, + LoggerFactory.CreateLogger()); + + var httpContext = new DefaultHttpContext(); + + await middleware.Invoke(httpContext); + + Assert.Contains(TestSink.Writes, w => w.Message.Contains("")); + } + private IOptionsMonitor CreateOptionsAccessor() { var options = new HttpLoggingOptions(); From 11023b8422a3f11c23f10100be00bf7cd247b7e5 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 10:25:25 -0700 Subject: [PATCH 29/45] Logging request body if it isn't logged --- .../HttpLogging/src/HttpLoggingMiddleware.cs | 14 +++++----- .../HttpLogging/src/RequestBufferingStream.cs | 26 ++++++++++++------- .../test/HttpLoggingMiddlewareTests.cs | 26 +++++++++++++++++++ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 3645d4394103..d9ee147f9842 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -129,13 +129,6 @@ public async Task Invoke(HttpContext context) try { - if ((HttpLoggingFields.Response & options.LoggingFields) == HttpLoggingFields.None) - { - // Short circuit and don't replace response body. - await _next(context); - return; - } - var response = context.Response; if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) @@ -154,6 +147,13 @@ public async Task Invoke(HttpContext context) await _next(context); + if (requestBufferingStream?.HasLogged == false) + { + // If the middleware pipeline didn't read until 0 was returned from readasync, + // make sure we log the request body. + requestBufferingStream.LogRequestBody(); + } + if (responseBufferingStream == null || responseBufferingStream.FirstWrite == false) { // No body, write headers here. diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index ef69dd6f358a..5d27f97c8348 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -18,7 +18,8 @@ internal sealed class RequestBufferingStream : BufferingStream { private Encoding? _encoding; private readonly int _limit; - private bool _hasLogged; + + public bool HasLogged { get; private set; } public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) : base(innerStream, logger) @@ -66,15 +67,10 @@ private void WriteToBuffer(ReadOnlySpan span, int res) return; } - if (res == 0 && !_hasLogged) + if (res == 0 && !HasLogged) { // Done reading, log the string. - var requestBody = GetString(_encoding); - if (requestBody != null) - { - _logger.RequestBody(requestBody); - } - _hasLogged = true; + LogRequestBody(); return; } @@ -91,15 +87,25 @@ private void WriteToBuffer(ReadOnlySpan span, int res) BuffersExtensions.Write(this, span.Slice(0, innerCount)); } - if (_limit - _bytesBuffered == 0 && !_hasLogged) + if (_limit - _bytesBuffered == 0 && !HasLogged) { var requestBody = GetString(_encoding); if (requestBody != null) { _logger.RequestBody(requestBody); } - _hasLogged = true; + HasLogged = true; + } + } + + public void LogRequestBody() + { + var requestBody = GetString(_encoding); + if (requestBody != null) + { + _logger.RequestBody(requestBody); } + HasLogged = true; } public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 8b09df4c8ec6..5ca7c86acd20 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -427,6 +427,32 @@ public async Task RequestBodyReadingLimitWorks() Assert.Contains(TestSink.Writes, w => w.Message.Equals("RequestBody: " + expected)); } + [Fact] + public async Task PartialReadBodyStillLogs() + { + var input = string.Concat(new string('a', 60000), new string('b', 3000)); + var options = CreateOptionsAccessor(); + options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; + + var middleware = new HttpLoggingMiddleware( + async c => + { + var arr = new byte[4096]; + var res = await c.Request.Body.ReadAsync(arr); + }, + options, + LoggerFactory.CreateLogger()); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.ContentType = "text/plain"; + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); + + await middleware.Invoke(httpContext); + var expected = input.Substring(0, 4096); + + Assert.Contains(TestSink.Writes, w => w.Message.Equals("RequestBody: " + expected)); + } + [Theory] [InlineData("text/plain")] [InlineData("text/html")] From 2f18049d4a460ad5256e45640c72128f99bc1d16 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 10:36:28 -0700 Subject: [PATCH 30/45] nit --- src/Middleware/HttpLogging/src/RequestBufferingStream.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index 5d27f97c8348..9316f4349a32 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -113,7 +113,6 @@ public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, Asy return TaskToApm.Begin(ReadAsync(buffer, offset, count), callback, state); } - /// public override int EndRead(IAsyncResult asyncResult) { return TaskToApm.End(asyncResult); From 93f39b7fe8b1a9c198aa698c73f7e9ed59da0616 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 10:37:54 -0700 Subject: [PATCH 31/45] Update src/Middleware/HttpLogging/src/HttpResponseLog.cs Co-authored-by: Kahbazi --- src/Middleware/HttpLogging/src/HttpResponseLog.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs index 0a9ec29b8825..89e0be6b004f 100644 --- a/src/Middleware/HttpLogging/src/HttpResponseLog.cs +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -26,7 +26,8 @@ public HttpResponseLog(List> keyValues) public IEnumerator> GetEnumerator() { - for (var i = 0; i < Count; i++) + var count = _keyValues.Count; + for (var i = 0; i < count; i++) { yield return _keyValues[i]; } From ba81e5102922bbdcf4c48a8ad558ddcca59f5fd8 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 12:32:50 -0700 Subject: [PATCH 32/45] Feedback --- .../Properties/launchSettings.json | 14 +++++++------- .../samples/HttpLogging.Sample/Startup.cs | 2 +- src/Middleware/HttpLogging/src/MediaTypeHelpers.cs | 2 +- src/Middleware/HttpLogging/src/MediaTypeOptions.cs | 3 ++- .../HttpLogging/src/RequestBufferingStream.cs | 4 ++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json index 2feff9ff305c..a5d93ea2eca4 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Properties/launchSettings.json @@ -1,4 +1,4 @@ -{ +{ "iisSettings": { "windowsAuthentication": false, "anonymousAuthentication": true, @@ -8,18 +8,18 @@ } }, "profiles": { - "IIS Express": { - "commandName": "IISExpress", + "HttpLogging.Sample": { + "commandName": "Project", + "dotnetRunMessages": "true", "launchBrowser": true, + "applicationUrl": "http://localhost:5000", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "HttpLogging.Sample": { - "commandName": "Project", - "dotnetRunMessages": "true", + "IIS Express": { + "commandName": "IISExpress", "launchBrowser": true, - "applicationUrl": "http://localhost:5000", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs index e23b8f6be1ee..9ee6252004c9 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs @@ -26,8 +26,8 @@ public void ConfigureServices(IServiceCollection services) // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - app.UseRouting(); app.UseHttpLogging(); + app.UseRouting(); app.UseEndpoints(endpoints => { endpoints.Map("/", async context => diff --git a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs index f38e4ec551d6..0537d682286b 100644 --- a/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs +++ b/src/Middleware/HttpLogging/src/MediaTypeHelpers.cs @@ -58,7 +58,7 @@ public static bool TryGetEncodingForMediaType(string contentType, List @@ -121,6 +121,7 @@ public MediaTypeState(MediaTypeHeaderValue mediaTypeHeaderValue) } public MediaTypeHeaderValue MediaTypeHeaderValue { get; } + public Encoding? Encoding { get; set; } public bool IsBinary { get; set; } } } diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index 9316f4349a32..b390de1f0a3e 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -16,12 +16,12 @@ namespace Microsoft.AspNetCore.HttpLogging { internal sealed class RequestBufferingStream : BufferingStream { - private Encoding? _encoding; + private Encoding _encoding; private readonly int _limit; public bool HasLogged { get; private set; } - public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding? encoding) + public RequestBufferingStream(Stream innerStream, int limit, ILogger logger, Encoding encoding) : base(innerStream, logger) { _logger = logger; From ea633802df7a50e12a88360199c03a98d93f98e3 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 13:06:03 -0700 Subject: [PATCH 33/45] Remove uneeded dep --- .../HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index cfd3a528bd45..221349a22dda 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -15,9 +15,9 @@ - + From 9327fc9f121b625a9b5977a3d6f44eb2ef65d495 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 13:30:33 -0700 Subject: [PATCH 34/45] Removing mroe --- .../HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index 221349a22dda..c3dbfe0085cb 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -14,9 +14,7 @@ - - From b9197c54f5e2a3d6446c94028d2121423c6b785e Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 13:33:26 -0700 Subject: [PATCH 35/45] Abstractions? --- .../HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index c3dbfe0085cb..93aeb7344b2e 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -13,7 +13,7 @@ - + From cb7fb4ec198d5844b223ad8c5026fb1754da1ceb Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 14:05:08 -0700 Subject: [PATCH 36/45] Targeting pack and comments --- src/Framework/test/TestData.cs | 1 + src/Middleware/HttpLogging/src/BufferingStream.cs | 10 ++++++++++ .../HttpLogging/test/HttpLoggingMiddlewareTests.cs | 7 ++++--- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Framework/test/TestData.cs b/src/Framework/test/TestData.cs index cf2d397e67b9..d388d4cbedcd 100644 --- a/src/Framework/test/TestData.cs +++ b/src/Framework/test/TestData.cs @@ -55,6 +55,7 @@ static TestData() "Microsoft.AspNetCore.Http.Connections.Common", "Microsoft.AspNetCore.Http.Extensions", "Microsoft.AspNetCore.Http.Features", + "Microsoft.AspNetCore.HttpLogging", "Microsoft.AspNetCore.HttpOverrides", "Microsoft.AspNetCore.HttpsPolicy", "Microsoft.AspNetCore.Identity", diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index c09b2e689d5d..78184f7fd3c6 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -76,8 +76,17 @@ public string GetString(Encoding? encoding) var bufferWriter = new ArrayBufferWriter(); var decoder = encoding.GetDecoder(); + // First calls convert on the entire ReadOnlySequence, with flush: false. + // flush: false is required as we don't want to write invalid characters that + // are spliced due to truncation. If we set flush: true, if effectively means + // we expect EOF in this array, meaning it will try to write any bytes at the end of it. EncodingExtensions.Convert(decoder, ros, bufferWriter, flush: false, out var charUsed, out var completed); + // Afterwards, we need to call convert in a loop until complete is true. + // The first call to convert many return true, but if it doesn't, we call + // Convert with a empty ReadOnlySequence and flush: true until we get completed: true. + + // This should never infinite due to the contract for decoders. while (true) { if (completed) @@ -124,6 +133,7 @@ public Span GetSpan(int sizeHint = 0) AllocateMemory(sizeHint); return _tailMemory.Span; } + private void AllocateMemory(int sizeHint) { if (_head == null) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 5ca7c86acd20..8ebb4c976092 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -356,9 +356,10 @@ public async Task RequestBodyReadingWorks(string expected) [Fact] public async Task RequestBodyReadingLimitLongCharactersWorks() { - var input = string.Concat(new string('あ', 60000)); + var input = string.Concat(new string('あ', 5)); var options = CreateOptionsAccessor(); options.CurrentValue.LoggingFields = HttpLoggingFields.RequestBody; + options.CurrentValue.RequestBodyLogLimit = 4; var middleware = new HttpLoggingMiddleware( async c => @@ -375,7 +376,7 @@ public async Task RequestBodyReadingLimitLongCharactersWorks() count += res; } - Assert.Equal(180000, count); + Assert.Equal(15, count); }, options, LoggerFactory.CreateLogger()); @@ -385,7 +386,7 @@ public async Task RequestBodyReadingLimitLongCharactersWorks() httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(input)); await middleware.Invoke(httpContext); - var expected = input.Substring(0, options.CurrentValue.ResponseBodyLogLimit / 3); + var expected = input.Substring(0, options.CurrentValue.RequestBodyLogLimit / 3); Assert.Contains(TestSink.Writes, w => w.Message.Equals("RequestBody: " + expected)); } From c1b2d2246f6735b3d75a778c474a90ece7751d7a Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 14:28:37 -0700 Subject: [PATCH 37/45] Extra check --- src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 8ebb4c976092..b1e223f934d8 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -531,6 +531,7 @@ public async Task RejectedContentTypes(string contentType) await middleware.Invoke(httpContext); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains(expected)); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("RequestBody: ")); } [Fact] From 0fe04246e7841e72f47a1caf7b225282f577a7a5 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 16:20:34 -0700 Subject: [PATCH 38/45] Fixing tests --- .../HttpLogging/src/BufferingStream.cs | 2 +- .../HttpLogging/src/HttpLoggingMiddleware.cs | 28 +++++++++++-------- .../test/HttpLoggingMiddlewareTests.cs | 4 +-- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 78184f7fd3c6..9e3dbf79ad2c 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -64,7 +64,7 @@ public string GetString(Encoding? encoding) if (encoding == null) { _logger.UnrecognizedMediaType(); - return ""; + return ""; } // Only place where we are actually using the buffered data. diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index d9ee147f9842..f2791476ed9f 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -101,18 +101,24 @@ public async Task Invoke(HttpContext context) FilterHeaders(list, request.Headers, options.RequestHeaders); } - if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody) - && MediaTypeHelpers.TryGetEncodingForMediaType(request.ContentType, + if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody)) + { + if (MediaTypeHelpers.TryGetEncodingForMediaType(request.ContentType, options.MediaTypeOptions.MediaTypeStates, out var encoding)) - { - originalBody = request.Body; - requestBufferingStream = new RequestBufferingStream( - request.Body, - options.RequestBodyLogLimit, - _logger, - encoding); - request.Body = requestBufferingStream; + { + originalBody = request.Body; + requestBufferingStream = new RequestBufferingStream( + request.Body, + options.RequestBodyLogLimit, + _logger, + encoding); + request.Body = requestBufferingStream; + } + else + { + _logger.UnrecognizedMediaType(); + } } var httpRequestLog = new HttpRequestLog(list); @@ -163,7 +169,7 @@ public async Task Invoke(HttpContext context) if (responseBufferingStream != null) { var responseBody = responseBufferingStream.GetString(responseBufferingStream.Encoding); - if (responseBody != null) + if (!string.IsNullOrEmpty(responseBody)) { _logger.ResponseBody(responseBody); } diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index b1e223f934d8..5f09ad5fc1b2 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -531,7 +531,7 @@ public async Task RejectedContentTypes(string contentType) await middleware.Invoke(httpContext); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains(expected)); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("RequestBody: ")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Unrecognized Content-Type for body.")); } [Fact] @@ -857,7 +857,7 @@ public async Task UnrecognizedMediaType() await middleware.Invoke(httpContext); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Unrecognized Content-Type for body.")); } private IOptionsMonitor CreateOptionsAccessor() From 7b13787583242a3420432dfae4243396f150086a Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 16:26:02 -0700 Subject: [PATCH 39/45] Fixing tests --- src/Framework/test/TestData.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Framework/test/TestData.cs b/src/Framework/test/TestData.cs index d388d4cbedcd..6df04b54e731 100644 --- a/src/Framework/test/TestData.cs +++ b/src/Framework/test/TestData.cs @@ -187,6 +187,7 @@ static TestData() { "Microsoft.AspNetCore.Http.Connections.Common", "6.0.0.0" }, { "Microsoft.AspNetCore.Http.Extensions", "6.0.0.0" }, { "Microsoft.AspNetCore.Http.Features", "6.0.0.0" }, + { "Microsoft.AspNetCore.HttpLogging", "6.0.0.0" }, { "Microsoft.AspNetCore.HttpOverrides", "6.0.0.0" }, { "Microsoft.AspNetCore.HttpsPolicy", "6.0.0.0" }, { "Microsoft.AspNetCore.Identity", "6.0.0.0" }, From e896579b1a7e06dfe2a451574d72051bff2ee81c Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 21:26:34 -0700 Subject: [PATCH 40/45] All of that feedback --- .../samples/HttpLogging.Sample/Program.cs | 7 +- .../samples/HttpLogging.Sample/Startup.cs | 5 - .../HttpLogging/src/BufferingStream.cs | 15 ++- .../src/HttpLoggingBuilderExtensions.cs | 4 +- .../HttpLogging/src/HttpLoggingFields.cs | 107 ++++++++++++++---- .../HttpLogging/src/HttpLoggingMiddleware.cs | 21 ++-- .../HttpLogging/src/HttpLoggingOptions.cs | 21 ++-- .../HttpLogging/src/HttpRequestLog.cs | 10 +- .../HttpLogging/src/HttpResponseLog.cs | 10 +- .../HttpLogging/src/MediaTypeOptions.cs | 15 ++- .../Microsoft.AspNetCore.HttpLogging.csproj | 1 - .../HttpLogging/src/RequestBufferingStream.cs | 7 +- .../test/HttpLoggingMiddlewareTests.cs | 5 +- .../test/HttpLoggingOptionsTests.cs | 2 +- 14 files changed, 144 insertions(+), 86 deletions(-) diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs index c4c215d505a8..2f43518e138f 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Program.cs @@ -1,12 +1,7 @@ +using System.Text.Json; using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; namespace HttpLogging.Sample { diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs index 9ee6252004c9..a85a95cd1b58 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs @@ -3,11 +3,6 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpLogging; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; namespace HttpLogging.Sample { diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 9e3dbf79ad2c..400ecab058a2 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -87,7 +87,8 @@ public string GetString(Encoding? encoding) // Convert with a empty ReadOnlySequence and flush: true until we get completed: true. // This should never infinite due to the contract for decoders. - while (true) + // But for safety, call this only 10 times, throwing a decode failure if it fails. + for (var i = 0; i < 10; i++) { if (completed) { @@ -98,6 +99,8 @@ public string GetString(Encoding? encoding) EncodingExtensions.Convert(decoder, ReadOnlySequence.Empty, bufferWriter, flush: true, out charUsed, out completed); } } + + throw new DecoderFallbackException("Failed to decode after 10 calls to Decoder.Convert"); } catch (DecoderFallbackException ex) { @@ -136,10 +139,10 @@ public Span GetSpan(int sizeHint = 0) private void AllocateMemory(int sizeHint) { - if (_head == null) + if (_head is null) { // We need to allocate memory to write since nobody has written before - BufferSegment newSegment = AllocateSegment(sizeHint); + var newSegment = AllocateSegment(sizeHint); // Set all the pointers _head = _tail = newSegment; @@ -147,7 +150,7 @@ private void AllocateMemory(int sizeHint) } else { - int bytesLeftInBuffer = _tailMemory.Length; + var bytesLeftInBuffer = _tailMemory.Length; if (bytesLeftInBuffer == 0 || bytesLeftInBuffer < sizeHint) { @@ -160,7 +163,7 @@ private void AllocateMemory(int sizeHint) _tailBytesBuffered = 0; } - BufferSegment newSegment = AllocateSegment(sizeHint); + var newSegment = AllocateSegment(sizeHint); _tail.SetNext(newSegment); _tail = newSegment; @@ -170,7 +173,7 @@ private void AllocateMemory(int sizeHint) private BufferSegment AllocateSegment(int sizeHint) { - BufferSegment newSegment = CreateSegment(); + var newSegment = CreateSegment(); // We can't use the recommended pool so use the ArrayPool newSegment.SetOwnedMemory(ArrayPool.Shared.Rent(GetSegmentSize(sizeHint))); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs index ae00cce9d466..81d5c1f4f5a0 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingBuilderExtensions.cs @@ -12,10 +12,10 @@ namespace Microsoft.AspNetCore.Builder public static class HttpLoggingBuilderExtensions { /// - /// Adds middleware for HTTP Logging. + /// Adds a middleware that can log HTTP requests and responses. /// /// The instance this method extends. - /// The for HttpLogging. + /// The . public static IApplicationBuilder UseHttpLogging(this IApplicationBuilder app) { if (app == null) diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index 867b92ee2432..ebda162a041c 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -2,107 +2,174 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; namespace Microsoft.AspNetCore.HttpLogging { /// - /// Logging enum to enable different request and response logging fields. + /// Flags used to control which parts of the + /// request and response are logged. /// [Flags] public enum HttpLoggingFields : long { /// - /// None + /// No logging. /// None = 0x0, /// - /// Request Path + /// Flag for logging the HTTP Request Path, which includes both the + /// and . + ///

+ /// For example: + /// Path: /index + /// PathBase: /app + ///

/// RequestPath = 0x1, /// - /// Request Query String + /// Flag for logging the HTTP Request . + ///

+ /// For example: + /// Query: ?index=1 + ///

/// RequestQuery = 0x2, /// - /// Request Protocol + /// Flag for logging the HTTP Request . + ///

+ /// For example: + /// Protocol: HTTP/1.1 + ///

/// RequestProtocol = 0x4, /// - /// Request Method + /// Flag for logging the HTTP Request . + ///

+ /// For example: + /// Method: GET + ///

/// RequestMethod = 0x8, /// - /// Request Scheme + /// Flag for logging the HTTP Request . + ///

+ /// For example: + /// Scheme: https + ///

/// RequestScheme = 0x10, /// - /// Response Status Code + /// Flag for logging the HTTP Response . + ///

+ /// For example: + /// StatusCode: 200 + ///

/// ResponseStatusCode = 0x20, /// - /// Request Headers + /// Flag for logging the HTTP Request . + /// Request Headers are logged as soon as the middleware is invoked. + /// Headers are redacted by default with the character 'X' unless specified in + /// the . + ///

+ /// For example: + /// Connection: keep-alive + /// My-Custom-Request-Header: X + ///

/// RequestHeaders = 0x40, /// - /// Response Headers + /// Flag for logging the HTTP Response . + /// Response Headers are logged when the is written to + /// or when + /// is called. + /// Headers are redacted by default with the character 'X' unless specified in + /// the . + ///

+ /// For example: + /// Content-Length: 16 + /// My-Custom-Response-Header: X + ///

/// ResponseHeaders = 0x80, /// - /// Request Trailers + /// Flag for logging the HTTP Request . + /// Request Trailers are currently not logged. /// RequestTrailers = 0x100, /// - /// Response Trailers + /// Flag for logging the HTTP Response . + /// Response Trailers are currently not logged. /// ResponseTrailers = 0x200, /// - /// Request Body + /// Flag for logging the HTTP Request . + /// Logging the request body has performance implications, as it requires buffering + /// the entire request body up to . /// RequestBody = 0x400, /// - /// Response Body + /// Flag for logging the HTTP Response . + /// Logging the response body has performance implications, as it requires buffering + /// the entire response body up to . /// ResponseBody = 0x800, /// - /// Combination of request properties, including Path, Query, Protocol, Method, and Scheme + /// Flag for logging a collection of HTTP Request properties, + /// including , , , + /// , and . /// RequestProperties = RequestPath | RequestQuery | RequestProtocol | RequestMethod | RequestScheme, /// - /// Combination of Request Properties and Request Headers + /// Flag for logging HTTP Request properties and headers. + /// Includes and /// RequestPropertiesAndHeaders = RequestProperties | RequestHeaders, /// - /// Combination of Response Properties and Response Headers + /// Flag for logging HTTP Response properties and headers. + /// Includes and /// ResponsePropertiesAndHeaders = ResponseStatusCode | ResponseHeaders, /// - /// Entire Request + /// Flag for logging the entire HTTP Request. + /// Includes and . + /// Logging the request body has performance implications, as it requires buffering + /// the entire request body up to . /// Request = RequestPropertiesAndHeaders | RequestBody, /// - /// Entire Response + /// Flag for logging the entire HTTP Response. + /// Includes and . + /// Logging the response body has performance implications, as it requires buffering + /// the entire response body up to . /// Response = ResponseStatusCode | ResponseHeaders | ResponseBody, /// - /// Entire Request and Response + /// Flag for logging both the HTTP Request and Response. + /// Includes and . + /// Logging the request and response body has performance implications, as it requires buffering + /// the entire request and response body up to the + /// and . /// All = Request | Response } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index f2791476ed9f..129b36d99589 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -5,6 +5,7 @@ using System.Buffers; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Text; using System.Threading; @@ -24,7 +25,7 @@ internal sealed class HttpLoggingMiddleware { private readonly RequestDelegate _next; private readonly ILogger _logger; - private IOptionsMonitor _options; + private readonly IOptionsMonitor _options; private const int DefaultRequestFieldsMinusHeaders = 7; private const int DefaultResponseFieldsMinusHeaders = 2; private const string Redacted = "X"; @@ -67,7 +68,7 @@ public async Task Invoke(HttpContext context) if ((HttpLoggingFields.Request & options.LoggingFields) != HttpLoggingFields.None) { var request = context.Request; - var list = new List>( + var list = new List>( request.Headers.Count + DefaultRequestFieldsMinusHeaders); if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestProtocol)) @@ -140,6 +141,7 @@ public async Task Invoke(HttpContext context) if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody)) { originalBodyFeature = context.Features.Get()!; + // TODO pool these. responseBufferingStream = new ResponseBufferingStream(originalBodyFeature, options.ResponseBodyLogLimit, @@ -193,19 +195,20 @@ public async Task Invoke(HttpContext context) } } - private static void AddToList(List> list, string key, string? value) + private static void AddToList(List> list, string key, string? value) { - list.Add(new KeyValuePair(key, value)); + list.Add(new KeyValuePair(key, value)); } public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions options, ILogger logger) { - var list = new List>( + var list = new List>( response.Headers.Count + DefaultResponseFieldsMinusHeaders); if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode)) { - list.Add(new KeyValuePair(nameof(response.StatusCode), response.StatusCode)); + list.Add(new KeyValuePair(nameof(response.StatusCode), + response.StatusCode.ToString(CultureInfo.InvariantCulture))); } if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) @@ -222,7 +225,7 @@ public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions formatter: HttpResponseLog.Callback); } - internal static void FilterHeaders(List> keyValues, + internal static void FilterHeaders(List> keyValues, IHeaderDictionary headers, ISet allowedHeaders) { @@ -231,10 +234,10 @@ internal static void FilterHeaders(List> keyValues if (!allowedHeaders.Contains(key)) { // Key is not among the "only listed" headers. - keyValues.Add(new KeyValuePair(key, Redacted)); + keyValues.Add(new KeyValuePair(key, Redacted)); continue; } - keyValues.Add(new KeyValuePair(key, value.ToString())); + keyValues.Add(new KeyValuePair(key, value.ToString())); } } } diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index f5051598206b..99c9c35b6c3f 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.HttpLogging { /// - /// Options for the + /// Options for the . /// public sealed class HttpLoggingOptions { @@ -20,11 +20,11 @@ public sealed class HttpLoggingOptions /// /// Request header values that are allowed to be logged. - /// - /// + ///

/// If a request header is not present in the , /// the header name will be logged with a redacted value. - /// + ///

+ /// public ISet RequestHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.Accept, @@ -40,11 +40,11 @@ public sealed class HttpLoggingOptions /// /// Response header values that are allowed to be logged. - /// - /// + ///

/// If a response header is not present in the , /// the header name will be logged with a redacted value. - /// + ///

+ /// public ISet ResponseHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.ContentLength, @@ -54,10 +54,11 @@ public sealed class HttpLoggingOptions /// /// Options for configuring encodings for a specific media type. + ///

+ /// If the request or response do not match the supported media type, + /// the response body will not be logged. + ///

/// - /// - /// If the request or response do not match the supported media type, the response body will not be logged. - /// public MediaTypeOptions MediaTypeOptions { get; } = MediaTypeOptions.BuildDefaultMediaTypeOptions(); /// diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs index 84c7234e554d..9c2a3873860d 100644 --- a/src/Middleware/HttpLogging/src/HttpRequestLog.cs +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -8,23 +8,23 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal sealed class HttpRequestLog : IReadOnlyList> + internal sealed class HttpRequestLog : IReadOnlyList> { - private readonly List> _keyValues; + private readonly List> _keyValues; private string? _cachedToString; internal static readonly Func Callback = (state, exception) => ((HttpRequestLog)state).ToString(); - public HttpRequestLog(List> keyValues) + public HttpRequestLog(List> keyValues) { _keyValues = keyValues; } - public KeyValuePair this[int index] => _keyValues[index]; + public KeyValuePair this[int index] => _keyValues[index]; public int Count => _keyValues.Count; - public IEnumerator> GetEnumerator() + public IEnumerator> GetEnumerator() { var count = _keyValues.Count; for (var i = 0; i < count; i++) diff --git a/src/Middleware/HttpLogging/src/HttpResponseLog.cs b/src/Middleware/HttpLogging/src/HttpResponseLog.cs index 89e0be6b004f..a82219f5d143 100644 --- a/src/Middleware/HttpLogging/src/HttpResponseLog.cs +++ b/src/Middleware/HttpLogging/src/HttpResponseLog.cs @@ -8,23 +8,23 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal sealed class HttpResponseLog : IReadOnlyList> + internal sealed class HttpResponseLog : IReadOnlyList> { - private readonly List> _keyValues; + private readonly List> _keyValues; private string? _cachedToString; internal static readonly Func Callback = (state, exception) => ((HttpResponseLog)state).ToString(); - public HttpResponseLog(List> keyValues) + public HttpResponseLog(List> keyValues) { _keyValues = keyValues; } - public KeyValuePair this[int index] => _keyValues[index]; + public KeyValuePair this[int index] => _keyValues[index]; public int Count => _keyValues.Count; - public IEnumerator> GetEnumerator() + public IEnumerator> GetEnumerator() { var count = _keyValues.Count; for (var i = 0; i < count; i++) diff --git a/src/Middleware/HttpLogging/src/MediaTypeOptions.cs b/src/Middleware/HttpLogging/src/MediaTypeOptions.cs index c05ff6792b30..f50e81e289b6 100644 --- a/src/Middleware/HttpLogging/src/MediaTypeOptions.cs +++ b/src/Middleware/HttpLogging/src/MediaTypeOptions.cs @@ -1,8 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + using System; using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; using Microsoft.Net.Http.Headers; namespace Microsoft.AspNetCore.HttpLogging @@ -12,7 +13,7 @@ namespace Microsoft.AspNetCore.HttpLogging /// public sealed class MediaTypeOptions { - private List _mediaTypeStates = new List(); + private readonly List _mediaTypeStates = new(); internal MediaTypeOptions() { @@ -113,16 +114,18 @@ public void Clear() _mediaTypeStates.Clear(); } - internal class MediaTypeState + internal readonly struct MediaTypeState { public MediaTypeState(MediaTypeHeaderValue mediaTypeHeaderValue) { MediaTypeHeaderValue = mediaTypeHeaderValue; + Encoding = null; + IsBinary = false; } public MediaTypeHeaderValue MediaTypeHeaderValue { get; } - public Encoding? Encoding { get; set; } - public bool IsBinary { get; set; } + public Encoding? Encoding { get; init; } + public bool IsBinary { get; init; } } } } diff --git a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj index 93aeb7344b2e..836949427bf6 100644 --- a/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj +++ b/src/Middleware/HttpLogging/src/Microsoft.AspNetCore.HttpLogging.csproj @@ -7,7 +7,6 @@ $(DefaultNetCoreTargetFramework) true true - aspnetcore;logging;http false enable diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index b390de1f0a3e..aae76d0f6bda 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -89,12 +89,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) if (_limit - _bytesBuffered == 0 && !HasLogged) { - var requestBody = GetString(_encoding); - if (requestBody != null) - { - _logger.RequestBody(requestBody); - } - HasLogged = true; + LogRequestBody(); } } diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index 5f09ad5fc1b2..d3c8f01edfe3 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -2,13 +2,10 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; using System.IO; using System.Text; -using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.HttpLogging; using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.Options; using Microsoft.Net.Http.Headers; @@ -16,7 +13,7 @@ using Moq; using Xunit; -namespace Microsoft.AspNetCore.HttpLogging.Tests +namespace Microsoft.AspNetCore.HttpLogging { public class HttpLoggingMiddlewareTests : LoggedTest { diff --git a/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs index e657516fe6f5..d91f23d2440b 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingOptionsTests.cs @@ -3,7 +3,7 @@ using Xunit; -namespace Microsoft.AspNetCore.HttpLogging.Tests +namespace Microsoft.AspNetCore.HttpLogging { public class HttpLoggingOptionsTests { From b472d318da928df9508ca774084bc899bc8ec6e4 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 22:20:36 -0700 Subject: [PATCH 41/45] Another round of feedback --- .../samples/HttpLogging.Sample/Startup.cs | 2 +- .../HttpLogging/src/BufferingStream.cs | 1 + .../HttpLogging/src/HttpLoggingExtensions.cs | 8 ++++---- .../HttpLogging/src/HttpLoggingMiddleware.cs | 10 +++++----- .../HttpLogging/src/HttpLoggingOptions.cs | 8 ++++++-- .../src/HttpLoggingServicesExtensions.cs | 4 +--- .../HttpLogging/src/LoggerEventIds.cs | 17 ----------------- .../HttpLogging/src/PublicAPI.Unshipped.txt | 2 +- .../HttpLogging/src/ResponseBufferingStream.cs | 13 +------------ 9 files changed, 20 insertions(+), 45 deletions(-) delete mode 100644 src/Middleware/HttpLogging/src/LoggerEventIds.cs diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs index a85a95cd1b58..77e7e6be254b 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs @@ -12,7 +12,7 @@ public class Startup // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { - services.AddHttpLogging(logging => + services.ConfigureHttpLogging(logging => { logging.LoggingFields = HttpLoggingFields.All; }); diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 400ecab058a2..720cec9c13ec 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -74,6 +74,7 @@ public string GetString(Encoding? encoding) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); var bufferWriter = new ArrayBufferWriter(); + encoding.GetString(ros); var decoder = encoding.GetDecoder(); // First calls convert on the entire ReadOnlySequence, with flush: false. diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs index 895a52c7f9ab..bd2c2706f860 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -9,16 +9,16 @@ namespace Microsoft.AspNetCore.HttpLogging internal static class HttpLoggingExtensions { private static readonly Action _requestBody = - LoggerMessage.Define(LogLevel.Information, LoggerEventIds.RequestBody, "RequestBody: {Body}"); + LoggerMessage.Define(LogLevel.Information, new EventId(3, "RequestBody"), "RequestBody: {Body}"); private static readonly Action _responseBody = - LoggerMessage.Define(LogLevel.Information, LoggerEventIds.ResponseBody, "ResponseBody: {Body}"); + LoggerMessage.Define(LogLevel.Information, new EventId(4, "ResponseBody"), "ResponseBody: {Body}"); private static readonly Action _decodeFailure = - LoggerMessage.Define(LogLevel.Debug, LoggerEventIds.DecodeFailure, "Decode failure while converting body."); + LoggerMessage.Define(LogLevel.Debug, new EventId(5, "DecodeFaulure"), "Decode failure while converting body."); private static readonly Action _unrecognizedMediaType = - LoggerMessage.Define(LogLevel.Debug, LoggerEventIds.UnrecognizedMediaType, "Unrecognized Content-Type for body."); + LoggerMessage.Define(LogLevel.Debug, new EventId(6, "UnrecognizedMediaType"), "Unrecognized Content-Type for body."); public static void RequestBody(this ILogger logger, string body) => _requestBody(logger, body, null); public static void ResponseBody(this ILogger logger, string body) => _responseBody(logger, body, null); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 129b36d99589..e08d260e51c9 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -99,7 +99,7 @@ public async Task Invoke(HttpContext context) if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders)) { - FilterHeaders(list, request.Headers, options.RequestHeaders); + FilterHeaders(list, request.Headers, options._internalRequestHeaders); } if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody)) @@ -125,7 +125,7 @@ public async Task Invoke(HttpContext context) var httpRequestLog = new HttpRequestLog(list); _logger.Log(LogLevel.Information, - eventId: LoggerEventIds.RequestLog, + eventId: new EventId(1, "RequestLog"), state: httpRequestLog, exception: null, formatter: HttpRequestLog.Callback); @@ -213,13 +213,13 @@ public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders)) { - FilterHeaders(list, response.Headers, options.ResponseHeaders); + FilterHeaders(list, response.Headers, options._internalResponseHeaders); } var httpResponseLog = new HttpResponseLog(list); logger.Log(LogLevel.Information, - eventId: LoggerEventIds.ResponseLog, + eventId: new EventId(2, "ResponseLog"), state: httpResponseLog, exception: null, formatter: HttpResponseLog.Callback); @@ -227,7 +227,7 @@ public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions internal static void FilterHeaders(List> keyValues, IHeaderDictionary headers, - ISet allowedHeaders) + HashSet allowedHeaders) { foreach (var (key, value) in headers) { diff --git a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs index 99c9c35b6c3f..800729b22aba 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingOptions.cs @@ -25,7 +25,9 @@ public sealed class HttpLoggingOptions /// the header name will be logged with a redacted value. ///

/// - public ISet RequestHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) + public ISet RequestHeaders => _internalRequestHeaders; + + internal HashSet _internalRequestHeaders = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.Accept, HeaderNames.AcceptEncoding, @@ -45,7 +47,9 @@ public sealed class HttpLoggingOptions /// the header name will be logged with a redacted value. ///

/// - public ISet ResponseHeaders { get; } = new HashSet(StringComparer.OrdinalIgnoreCase) + public ISet ResponseHeaders => _internalResponseHeaders; + + internal HashSet _internalResponseHeaders = new HashSet(StringComparer.OrdinalIgnoreCase) { HeaderNames.ContentLength, HeaderNames.ContentType, diff --git a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs index c057805eaf10..a6fad2d4106b 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs @@ -3,8 +3,6 @@ using System; using Microsoft.AspNetCore.HttpLogging; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; namespace Microsoft.Extensions.DependencyInjection { @@ -19,7 +17,7 @@ public static class HttpLoggingServicesExtensions /// The for adding services. /// A delegate to configure the . /// - public static IServiceCollection AddHttpLogging(this IServiceCollection services, Action configureOptions) + public static IServiceCollection ConfigureHttpLogging(this IServiceCollection services, Action configureOptions) { if (services == null) { diff --git a/src/Middleware/HttpLogging/src/LoggerEventIds.cs b/src/Middleware/HttpLogging/src/LoggerEventIds.cs deleted file mode 100644 index a1d156953738..000000000000 --- a/src/Middleware/HttpLogging/src/LoggerEventIds.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using Microsoft.Extensions.Logging; - -namespace Microsoft.AspNetCore.HttpLogging -{ - internal static class LoggerEventIds - { - public static readonly EventId RequestLog = new EventId(1, "RequestLog"); - public static readonly EventId ResponseLog = new EventId(2, "ResponseLog"); - public static readonly EventId RequestBody = new EventId(3, "RequestBody"); - public static readonly EventId ResponseBody = new EventId(4, "ResponseBody"); - public static readonly EventId DecodeFailure = new EventId(5, "DecodeFaulure"); - public static readonly EventId UnrecognizedMediaType = new EventId(6, "UnrecognizedMediaType"); - } -} diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index 986d071efcef..b4e5f541165c 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -40,4 +40,4 @@ Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType, S Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.Clear() -> void Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! -static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.ConfigureHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index ec0fbc002eb4..f0880654ccb8 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -55,18 +55,7 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, public Stream Stream => this; - public PipeWriter Writer - { - get - { - if (_pipeAdapter == null) - { - _pipeAdapter = PipeWriter.Create(Stream, _pipeWriterOptions); - } - - return _pipeAdapter; - } - } + public PipeWriter Writer => _pipeAdapter ??= PipeWriter.Create(Stream, _pipeWriterOptions) public Encoding? Encoding { get => _encoding; } From 26d75f2dde5454f9e7ccbaa3894ee564ba5bab0b Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 19 Apr 2021 22:28:31 -0700 Subject: [PATCH 42/45] Override writeasync --- src/Middleware/HttpLogging/src/BufferingStream.cs | 10 ++++++++++ .../HttpLogging/src/ResponseBufferingStream.cs | 15 ++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 720cec9c13ec..26fd977e2700 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -269,6 +269,16 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati return _innerStream.WriteAsync(buffer, offset, count, cancellationToken); } + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + return _innerStream.WriteAsync(buffer, cancellationToken); + } + + public override void Write(ReadOnlySpan buffer) + { + _innerStream.Write(buffer); + } + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) { return _innerStream.BeginRead(buffer, offset, count, callback, state); diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index f0880654ccb8..bedbb89034a1 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -55,7 +55,7 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature, public Stream Stream => this; - public PipeWriter Writer => _pipeAdapter ??= PipeWriter.Create(Stream, _pipeWriterOptions) + public PipeWriter Writer => _pipeAdapter ??= PipeWriter.Create(Stream, _pipeWriterOptions); public Encoding? Encoding { get => _encoding; } @@ -96,25 +96,30 @@ public override void Write(ReadOnlySpan span) } public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await WriteAsync(new Memory(buffer, offset, count), cancellationToken); + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) { var remaining = _limit - _bytesBuffered; - var innerCount = Math.Min(remaining, count); + var innerCount = Math.Min(remaining, buffer.Length); OnFirstWrite(); if (_tailMemory.Length - innerCount > 0) { - buffer.AsSpan(offset, count).CopyTo(_tailMemory.Span); + buffer.Slice(0, innerCount).CopyTo(_tailMemory); _tailBytesBuffered += innerCount; _bytesBuffered += innerCount; _tailMemory = _tailMemory.Slice(innerCount); } else { - BuffersExtensions.Write(this, buffer.AsSpan(offset, innerCount)); + BuffersExtensions.Write(this, buffer.Span); } - await _innerStream.WriteAsync(buffer, offset, count, cancellationToken); + await _innerStream.WriteAsync(buffer, cancellationToken); } private void OnFirstWrite() From 8b3f6106d2f2360681066b2f89e146aab85c29b7 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 20 Apr 2021 10:47:26 -0700 Subject: [PATCH 43/45] Feedback --- .../samples/HttpLogging.Sample/Startup.cs | 2 +- .../HttpLogging/src/BufferingStream.cs | 2 +- .../HttpLogging/src/HttpLoggingExtensions.cs | 13 ++++++++++++ .../HttpLogging/src/HttpLoggingFields.cs | 8 ++++---- .../HttpLogging/src/HttpLoggingMiddleware.cs | 20 +++++++++---------- .../src/HttpLoggingServicesExtensions.cs | 2 +- .../HttpLogging/src/HttpRequestLog.cs | 1 + .../HttpLogging/src/MediaTypeOptions.cs | 6 +----- .../HttpLogging/src/PublicAPI.Unshipped.txt | 3 +-- .../HttpLogging/src/RequestBufferingStream.cs | 10 +++++----- .../test/HttpLoggingMiddlewareTests.cs | 10 +++++----- 11 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs index 77e7e6be254b..a85a95cd1b58 100644 --- a/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs +++ b/src/Middleware/HttpLogging/samples/HttpLogging.Sample/Startup.cs @@ -12,7 +12,7 @@ public class Startup // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { - services.ConfigureHttpLogging(logging => + services.AddHttpLogging(logging => { logging.LoggingFields = HttpLoggingFields.All; }); diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index 26fd977e2700..d5cf0f0217bb 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -14,7 +14,7 @@ namespace Microsoft.AspNetCore.HttpLogging { - internal class BufferingStream : Stream, IBufferWriter + internal abstract class BufferingStream : Stream, IBufferWriter { private const int MinimumBufferSize = 4096; // 4K protected int _bytesBuffered; diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs index bd2c2706f860..47f0429041be 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -20,6 +20,19 @@ internal static class HttpLoggingExtensions private static readonly Action _unrecognizedMediaType = LoggerMessage.Define(LogLevel.Debug, new EventId(6, "UnrecognizedMediaType"), "Unrecognized Content-Type for body."); + + public static void RequestLog(this ILogger logger, HttpRequestLog requestLog) => logger.Log( + LogLevel.Information, + new EventId(1, "RequestLogLog"), + requestLog, + exception: null, + formatter: HttpRequestLog.Callback); + public static void ResponseLog(this ILogger logger, HttpResponseLog responseLog) => logger.Log( + LogLevel.Information, + new EventId(2, "ResponseLog"), + responseLog, + exception: null, + formatter: HttpResponseLog.Callback); public static void RequestBody(this ILogger logger, string body) => _requestBody(logger, body, null); public static void ResponseBody(this ILogger logger, string body) => _responseBody(logger, body, null); public static void DecodeFailure(this ILogger logger, Exception ex) => _decodeFailure(logger, ex); diff --git a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs index ebda162a041c..dab6c3208491 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingFields.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingFields.cs @@ -78,12 +78,12 @@ public enum HttpLoggingFields : long /// /// Flag for logging the HTTP Request . /// Request Headers are logged as soon as the middleware is invoked. - /// Headers are redacted by default with the character 'X' unless specified in + /// Headers are redacted by default with the character '[Redacted]' unless specified in /// the . ///

/// For example: /// Connection: keep-alive - /// My-Custom-Request-Header: X + /// My-Custom-Request-Header: [Redacted] ///

/// RequestHeaders = 0x40, @@ -93,12 +93,12 @@ public enum HttpLoggingFields : long /// Response Headers are logged when the is written to /// or when /// is called. - /// Headers are redacted by default with the character 'X' unless specified in + /// Headers are redacted by default with the character '[Redacted]' unless specified in /// the . ///

/// For example: /// Content-Length: 16 - /// My-Custom-Response-Header: X + /// My-Custom-Response-Header: [Redacted] ///

/// ResponseHeaders = 0x80, diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index e08d260e51c9..4952ef405d40 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -28,7 +28,7 @@ internal sealed class HttpLoggingMiddleware private readonly IOptionsMonitor _options; private const int DefaultRequestFieldsMinusHeaders = 7; private const int DefaultResponseFieldsMinusHeaders = 2; - private const string Redacted = "X"; + private const string Redacted = "[Redacted]"; /// /// Initializes . @@ -61,6 +61,12 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptionsMonitorHttpResponseLog.cs public async Task Invoke(HttpContext context) { + if (!_logger.IsEnabled(LogLevel.Information)) + { + // Logger isn't enabled. + await _next(context); + return; + } var options = _options.CurrentValue; RequestBufferingStream? requestBufferingStream = null; Stream? originalBody = null; @@ -124,11 +130,7 @@ public async Task Invoke(HttpContext context) var httpRequestLog = new HttpRequestLog(list); - _logger.Log(LogLevel.Information, - eventId: new EventId(1, "RequestLog"), - state: httpRequestLog, - exception: null, - formatter: HttpRequestLog.Callback); + _logger.RequestLog(httpRequestLog); } ResponseBufferingStream? responseBufferingStream = null; @@ -218,11 +220,7 @@ public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions var httpResponseLog = new HttpResponseLog(list); - logger.Log(LogLevel.Information, - eventId: new EventId(2, "ResponseLog"), - state: httpResponseLog, - exception: null, - formatter: HttpResponseLog.Callback); + logger.ResponseLog(httpResponseLog); } internal static void FilterHeaders(List> keyValues, diff --git a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs index a6fad2d4106b..1d755dcfc8ac 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingServicesExtensions.cs @@ -17,7 +17,7 @@ public static class HttpLoggingServicesExtensions /// The for adding services. /// A delegate to configure the . /// - public static IServiceCollection ConfigureHttpLogging(this IServiceCollection services, Action configureOptions) + public static IServiceCollection AddHttpLogging(this IServiceCollection services, Action configureOptions) { if (services == null) { diff --git a/src/Middleware/HttpLogging/src/HttpRequestLog.cs b/src/Middleware/HttpLogging/src/HttpRequestLog.cs index 9c2a3873860d..2dd608e4c32e 100644 --- a/src/Middleware/HttpLogging/src/HttpRequestLog.cs +++ b/src/Middleware/HttpLogging/src/HttpRequestLog.cs @@ -37,6 +37,7 @@ public override string ToString() { if (_cachedToString == null) { + // TODO use string.Create instead of a StringBuilder here. var builder = new StringBuilder(); var count = _keyValues.Count; builder.Append("Request:"); diff --git a/src/Middleware/HttpLogging/src/MediaTypeOptions.cs b/src/Middleware/HttpLogging/src/MediaTypeOptions.cs index f50e81e289b6..beb5314ef2d4 100644 --- a/src/Middleware/HttpLogging/src/MediaTypeOptions.cs +++ b/src/Middleware/HttpLogging/src/MediaTypeOptions.cs @@ -33,11 +33,7 @@ internal static MediaTypeOptions BuildDefaultMediaTypeOptions() return options; } - /// - /// Adds a to be used for logging as text. - /// - /// The MediaType to add. - public void AddText(MediaTypeHeaderValue mediaType) + internal void AddText(MediaTypeHeaderValue mediaType) { if (mediaType == null) { diff --git a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt index b4e5f541165c..8295c237914a 100644 --- a/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt +++ b/src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt @@ -34,10 +34,9 @@ Microsoft.AspNetCore.HttpLogging.HttpLoggingOptions.ResponseHeaders.get -> Syste Microsoft.AspNetCore.HttpLogging.MediaTypeOptions Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddBinary(Microsoft.Net.Http.Headers.MediaTypeHeaderValue! mediaType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddBinary(string! contentType) -> void -Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(Microsoft.Net.Http.Headers.MediaTypeHeaderValue! mediaType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.AddText(string! contentType, System.Text.Encoding! encoding) -> void Microsoft.AspNetCore.HttpLogging.MediaTypeOptions.Clear() -> void Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions static Microsoft.AspNetCore.Builder.HttpLoggingBuilderExtensions.UseHttpLogging(this Microsoft.AspNetCore.Builder.IApplicationBuilder! app) -> Microsoft.AspNetCore.Builder.IApplicationBuilder! -static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.ConfigureHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Microsoft.Extensions.DependencyInjection.HttpLoggingServicesExtensions.AddHttpLogging(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action! configureOptions) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs index aae76d0f6bda..c9ef64584c92 100644 --- a/src/Middleware/HttpLogging/src/RequestBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/RequestBufferingStream.cs @@ -34,7 +34,7 @@ public override async ValueTask ReadAsync(Memory destination, Cancell { var res = await _innerStream.ReadAsync(destination, cancellationToken); - WriteToBuffer(destination.Slice(0, res).Span, res); + WriteToBuffer(destination.Slice(0, res).Span); return res; } @@ -43,7 +43,7 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, { var res = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken); - WriteToBuffer(buffer.AsSpan(offset, res), res); + WriteToBuffer(buffer.AsSpan(offset, res)); return res; } @@ -52,12 +52,12 @@ public override int Read(byte[] buffer, int offset, int count) { var res = _innerStream.Read(buffer, offset, count); - WriteToBuffer(buffer.AsSpan(offset, res), res); + WriteToBuffer(buffer.AsSpan(offset, res)); return res; } - private void WriteToBuffer(ReadOnlySpan span, int res) + private void WriteToBuffer(ReadOnlySpan span) { // get what was read into the buffer var remaining = _limit - _bytesBuffered; @@ -67,7 +67,7 @@ private void WriteToBuffer(ReadOnlySpan span, int res) return; } - if (res == 0 && !HasLogged) + if (span.Length == 0 && !HasLogged) { // Done reading, log the string. LogRequestBody(); diff --git a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs index d3c8f01edfe3..710e6ad701b4 100644 --- a/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs +++ b/src/Middleware/HttpLogging/test/HttpLoggingMiddlewareTests.cs @@ -286,7 +286,7 @@ public async Task UnknownRequestHeadersRedacted() httpContext.Request.Headers["foo"] = "bar"; await middleware.Invoke(httpContext); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: X")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: [Redacted]")); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("foo: bar")); } @@ -313,8 +313,8 @@ public async Task CanConfigureRequestAllowList() await middleware.Invoke(httpContext); Assert.Contains(TestSink.Writes, w => w.Message.Contains("foo: bar")); - Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("foo: X")); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: X")); + Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("foo: [Redacted]")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Connection: [Redacted]")); Assert.DoesNotContain(TestSink.Writes, w => w.Message.Contains("Connection: keep-alive")); } @@ -686,7 +686,7 @@ public async Task ResponseHeadersRedacted() var httpContext = new DefaultHttpContext(); await middleware.Invoke(httpContext); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("Test: X")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Test: [Redacted]")); } [Fact] @@ -711,7 +711,7 @@ public async Task AllowedResponseHeadersModify() await middleware.Invoke(httpContext); Assert.Contains(TestSink.Writes, w => w.Message.Contains("Test: Kestrel")); - Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: X")); + Assert.Contains(TestSink.Writes, w => w.Message.Contains("Server: [Redacted]")); } [Theory] From 10736052b4d40466068213e4d8bec7fb11aa8078 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 20 Apr 2021 11:54:44 -0700 Subject: [PATCH 44/45] Fixing some small parts --- src/Middleware/HttpLogging/src/BufferingStream.cs | 1 - .../HttpLogging/src/HttpLoggingExtensions.cs | 1 - .../HttpLogging/src/HttpLoggingMiddleware.cs | 11 ++++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Middleware/HttpLogging/src/BufferingStream.cs b/src/Middleware/HttpLogging/src/BufferingStream.cs index d5cf0f0217bb..90ee3b01a7d8 100644 --- a/src/Middleware/HttpLogging/src/BufferingStream.cs +++ b/src/Middleware/HttpLogging/src/BufferingStream.cs @@ -74,7 +74,6 @@ public string GetString(Encoding? encoding) var ros = new ReadOnlySequence(_head, 0, _tail, _tailBytesBuffered); var bufferWriter = new ArrayBufferWriter(); - encoding.GetString(ros); var decoder = encoding.GetDecoder(); // First calls convert on the entire ReadOnlySequence, with flush: false. diff --git a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs index 47f0429041be..523c02efde9e 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingExtensions.cs @@ -20,7 +20,6 @@ internal static class HttpLoggingExtensions private static readonly Action _unrecognizedMediaType = LoggerMessage.Define(LogLevel.Debug, new EventId(6, "UnrecognizedMediaType"), "Unrecognized Content-Type for body."); - public static void RequestLog(this ILogger logger, HttpRequestLog requestLog) => logger.Log( LogLevel.Information, new EventId(1, "RequestLogLog"), diff --git a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs index 4952ef405d40..ba56129f14b7 100644 --- a/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs +++ b/src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs @@ -59,14 +59,19 @@ public HttpLoggingMiddleware(RequestDelegate next, IOptionsMonitor /// /// HttpResponseLog.cs - public async Task Invoke(HttpContext context) + public Task Invoke(HttpContext context) { if (!_logger.IsEnabled(LogLevel.Information)) { // Logger isn't enabled. - await _next(context); - return; + return _next(context); } + + return InvokeInternal(context); + } + + private async Task InvokeInternal(HttpContext context) + { var options = _options.CurrentValue; RequestBufferingStream? requestBufferingStream = null; Stream? originalBody = null; From fc0073d2a4f13dd250893e5da04ea3abbaf09048 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 20 Apr 2021 12:11:28 -0700 Subject: [PATCH 45/45] Fixing response buffering check --- .../src/ResponseBufferingStream.cs | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs index bedbb89034a1..3c889b92ce1d 100644 --- a/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs +++ b/src/Middleware/HttpLogging/src/ResponseBufferingStream.cs @@ -81,15 +81,18 @@ public override void Write(ReadOnlySpan span) OnFirstWrite(); - if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) - { - _tailBytesBuffered += innerCount; - _bytesBuffered += innerCount; - _tailMemory = _tailMemory.Slice(innerCount); - } - else - { - BuffersExtensions.Write(this, span.Slice(0, innerCount)); + if (innerCount > 0) + { + if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span)) + { + _tailBytesBuffered += innerCount; + _bytesBuffered += innerCount; + _tailMemory = _tailMemory.Slice(innerCount); + } + else + { + BuffersExtensions.Write(this, span.Slice(0, innerCount)); + } } _innerStream.Write(span); @@ -107,16 +110,19 @@ public override async ValueTask WriteAsync(ReadOnlyMemory buffer, Cancella OnFirstWrite(); - if (_tailMemory.Length - innerCount > 0) - { - buffer.Slice(0, innerCount).CopyTo(_tailMemory); - _tailBytesBuffered += innerCount; - _bytesBuffered += innerCount; - _tailMemory = _tailMemory.Slice(innerCount); - } - else + if (innerCount > 0) { - BuffersExtensions.Write(this, buffer.Span); + if (_tailMemory.Length - innerCount > 0) + { + buffer.Slice(0, innerCount).CopyTo(_tailMemory); + _tailBytesBuffered += innerCount; + _bytesBuffered += innerCount; + _tailMemory = _tailMemory.Slice(innerCount); + } + else + { + BuffersExtensions.Write(this, buffer.Span); + } } await _innerStream.WriteAsync(buffer, cancellationToken);