Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve Kestrel connection metrics with error reasons #55565

Merged
merged 43 commits into from
Jul 22, 2024
Merged
Changes from 1 commit
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
d62e9a2
Add protocol code to HTTP/2 and HTTP/3 connection metric
JamesNK Apr 25, 2024
32484b3
Add reason to connection exception
JamesNK May 7, 2024
121c074
More
JamesNK May 7, 2024
52f5fd4
Clean up
JamesNK May 7, 2024
98c752b
HTTP/1.1 and HTTP/2
JamesNK May 7, 2024
c6e4efa
Tests
JamesNK May 8, 2024
6b91a77
Fix build
JamesNK May 8, 2024
de8499b
Fix microbenchmarks
JamesNK May 8, 2024
165f8e9
Tests
JamesNK May 9, 2024
e2c9aa7
Rename
JamesNK May 9, 2024
16744ac
Fix tests
JamesNK May 9, 2024
400dde9
Clean up
JamesNK May 9, 2024
576c1d2
Remove NoError and always tag reason tag
JamesNK May 9, 2024
a4c1b32
Fix build
JamesNK May 9, 2024
6f8ecbf
Report error from failed TLS handshake
JamesNK May 10, 2024
07d7fa7
Fix build
JamesNK May 10, 2024
4bdd15d
Fix merge
JamesNK Jun 10, 2024
6ac22e1
Snake case tag value
JamesNK Jun 10, 2024
cd3aa00
Fix build
JamesNK Jun 10, 2024
beee7fa
Update
JamesNK Jun 10, 2024
88b4c8c
Update
JamesNK Jun 10, 2024
193e4a1
Tests, graceful shutdown vs abort shutdown, add reason for invalid bo…
JamesNK Jun 11, 2024
d7825d6
WIP
JamesNK Jun 15, 2024
733476a
HTTP/1.1 end reason work
JamesNK Jun 18, 2024
8e1fd7f
Comment
JamesNK Jun 18, 2024
6bb6196
Add reasons for some HTTP/1.1 request body errors
JamesNK Jun 28, 2024
8fb7354
More tests
JamesNK Jun 30, 2024
077cf06
Add exceeded max concurrent connections reason
JamesNK Jul 1, 2024
33a98af
Flaky test
JamesNK Jul 1, 2024
47f5ebe
WIP
JamesNK Jul 3, 2024
fac7e1c
WIP
JamesNK Jul 4, 2024
b90be50
Update
JamesNK Jul 4, 2024
cc3a26f
Fix build
JamesNK Jul 4, 2024
34f175b
Clean up
JamesNK Jul 4, 2024
7b517ea
PR feedback
JamesNK Jul 17, 2024
38f887c
PR feedback
JamesNK Jul 18, 2024
d066571
PR feedback
JamesNK Jul 19, 2024
550376d
Add header limits exceeded
JamesNK Jul 19, 2024
e51f962
Fix tests
JamesNK Jul 19, 2024
8238159
Fix tests
JamesNK Jul 19, 2024
43f692e
PR feedback
JamesNK Jul 20, 2024
63a604e
Fix test
JamesNK Jul 20, 2024
0ac31b2
Fix test
JamesNK Jul 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
WIP
JamesNK committed Jul 17, 2024
commit fac7e1c4bece4658e13899c22bff4017e4fbb008
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@
using System.Diagnostics;
using System.IO.Pipelines;
using Microsoft.AspNetCore.Connections;
using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;

namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;

@@ -136,6 +137,7 @@ private async Task PumpAsync()
// Read() will have already have greedily consumed the entire request body if able.
if (result.IsCompleted)
{
KestrelMetrics.AddConnectionEndReason(_context.MetricsContext, ConnectionEndReason.UnexpectedEndOfRequestContent);
ThrowUnexpectedEndOfRequestContent();
}
}
53 changes: 27 additions & 26 deletions src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs
Original file line number Diff line number Diff line change
@@ -229,7 +229,6 @@ bool TrimAndTakeStartLine(ref SequenceReader<byte> reader)
if (!_parser.ParseRequestLine(new Http1ParsingHandler(this), ref trimmedReader))
{
// We read the maximum allowed but didn't complete the start line.
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestLine);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.RequestLineTooLong);
}

@@ -273,7 +272,6 @@ bool TrimAndTakeMessageHeaders(ref SequenceReader<byte> reader, bool trailers)
if (!_parser.ParseHeaders(new Http1ParsingHandler(this, trailers), ref trimmedReader))
{
// We read the maximum allowed but didn't complete the headers.
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.HeadersExceedMaxTotalSize);
}

@@ -608,12 +606,10 @@ internal void EnsureHostHeaderExists()
return;
}

KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.MissingHostHeader);
}
else if (hostCount > 1)
{
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.MultipleHostHeaders);
}
else if (_requestTargetForm != HttpRequestTarget.OriginForm)
@@ -722,7 +718,6 @@ protected override bool TryParseRequest(ReadResult result, out bool endConnectio
}
catch (InvalidOperationException) when (_requestProcessingStatus == RequestProcessingStatus.ParsingHeaders)
{
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.MalformedRequestInvalidHeaders);
throw;
}
@@ -751,11 +746,9 @@ protected override bool TryParseRequest(ReadResult result, out bool endConnectio
endConnection = true;
return true;
case RequestProcessingStatus.ParsingRequestLine:
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.InvalidRequestLine);
break;
case RequestProcessingStatus.ParsingHeaders:
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.MalformedRequestInvalidHeaders);
break;
}
@@ -771,7 +764,6 @@ protected override bool TryParseRequest(ReadResult result, out bool endConnectio
{
// In this case, there is an ongoing request but the start line/header parsing has timed out, so send
// a 408 response.
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.RequestHeadersTimeout);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.RequestHeadersTimeout);
}

@@ -787,26 +779,20 @@ protected override bool TryParseRequest(ReadResult result, out bool endConnectio
}
}

internal static ConnectionEndReason GetConnectionEndReason(Microsoft.AspNetCore.Http.BadHttpRequestException ex)
{
#pragma warning disable CS0618 // Type or member is obsolete
private void OnBadRequest(ReadOnlySequence<byte> requestData, BadHttpRequestException ex)
var kestrelEx = ex as BadHttpRequestException;
#pragma warning restore CS0618 // Type or member is obsolete
{
var reason = ex.Reason;

// Some code shared between HTTP versions throws errors. For example, HttpRequestHeaders collection
// throws when an invalid content length is set.
// Only want to set a reasons for HTTP/1.1 connection, so set end reason by catching the exception here.
switch (reason)
switch (kestrelEx?.Reason)
{
case RequestRejectionReason.UnrecognizedHTTPVersion:
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidHttpVersion);
DetectHttp2Preface(requestData);
break;
return ConnectionEndReason.InvalidHttpVersion;
case RequestRejectionReason.InvalidRequestLine:
case RequestRejectionReason.RequestLineTooLong:
case RequestRejectionReason.InvalidRequestTarget:
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestLine);
break;
return ConnectionEndReason.InvalidRequestLine;
case RequestRejectionReason.InvalidRequestHeadersNoCRLF:
case RequestRejectionReason.InvalidRequestHeader:
case RequestRejectionReason.InvalidContentLength:
@@ -821,16 +807,31 @@ private void OnBadRequest(ReadOnlySequence<byte> requestData, BadHttpRequestExce
case RequestRejectionReason.MissingHostHeader:
case RequestRejectionReason.MultipleHostHeaders:
case RequestRejectionReason.InvalidHostHeader:
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.InvalidRequestHeaders);
break;
return ConnectionEndReason.InvalidRequestHeaders;
case RequestRejectionReason.TlsOverHttpError:
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.TlsOverHttp);
break;
return ConnectionEndReason.TlsOverHttp;
case RequestRejectionReason.UnexpectedEndOfRequestContent:
return ConnectionEndReason.UnexpectedEndOfRequestContent;
default:
// In some scenarios the end reason might already be set to a more specific error
// and attempting to set the reason again has no impact.
KestrelMetrics.AddConnectionEndReason(MetricsContext, ConnectionEndReason.OtherError);
break;
return ConnectionEndReason.OtherError;
}
}

#pragma warning disable CS0618 // Type or member is obsolete
private void OnBadRequest(ReadOnlySequence<byte> requestData, BadHttpRequestException ex)
#pragma warning restore CS0618 // Type or member is obsolete
{
// Some code shared between HTTP versions throws errors. For example, HttpRequestHeaders collection
// throws when an invalid content length is set.
// Only want to set a reasons for HTTP/1.1 connection, so set end reason by catching the exception here.
var reason = GetConnectionEndReason(ex);
KestrelMetrics.AddConnectionEndReason(MetricsContext, reason);

if (ex.Reason == RequestRejectionReason.UnrecognizedHTTPVersion)
{
DetectHttp2Preface(requestData);
}
}

Original file line number Diff line number Diff line change
@@ -270,7 +270,6 @@ private void VerifyIsNotReading()
{
if (_readResult.IsCompleted)
{
KestrelMetrics.AddConnectionEndReason(_context.MetricsContext, ConnectionEndReason.UnexpectedEndOfRequestContent);
KestrelBadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent);
}

Original file line number Diff line number Diff line change
@@ -1408,8 +1408,7 @@ public void SetBadRequestState(BadHttpRequestException ex)
WriteDiagnosticEvent(ServiceContext.DiagnosticSource, badRequestEventName, this);
}

// TODO: A specific error should be set. Pass unset here
DisableKeepAlive(ConnectionEndReason.Unexpected);
DisableKeepAlive(Http1Connection.GetConnectionEndReason(ex));
}

internal virtual void DisableKeepAlive(ConnectionEndReason reason)