HTTP2 Dynamic Window (prototype/early draft)

src/libraries/Common/src/Interop/Windows/WinSock/Interop.WSAIoctl.TcpInfo.cs

@@ -0,0 +1,55 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Net.Sockets;
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+ internal static partial class Winsock
+ {
+ private const int SioTcpInfo = unchecked((int)3623878695L);
+
+ [DllImport(Interop.Libraries.Ws2_32, SetLastError = true, EntryPoint = "WSAIoctl")]
+ private static extern SocketError WSAIoctl_Blocking(
+ SafeSocketHandle socketHandle,
+ [In] int ioControlCode,
+ [In] ref int inBuffer,
+ [In] int inBufferSize,
+ [Out] out _TCP_INFO_v0 outBuffer,
+ [In] int outBufferSize,
+ [Out] out int bytesTransferred,
+ [In] IntPtr overlapped,
+ [In] IntPtr completionRoutine);
+
+ internal static unsafe SocketError GetTcpInfoV0(SafeSocketHandle socketHandle, out _TCP_INFO_v0 tcpInfo)
+ {
+ int input = 0;
+ return WSAIoctl_Blocking(socketHandle, SioTcpInfo, ref input, sizeof(int), out tcpInfo, sizeof(_TCP_INFO_v0), out _, IntPtr.Zero, IntPtr.Zero);
+ }
+
+ internal struct _TCP_INFO_v0
+ {
+ internal System.Net.NetworkInformation.TcpState State;
+ internal uint Mss;
+ internal ulong ConnectionTimeMs;
+ internal byte TimestampsEnabled;
+ internal uint RttUs;
+ internal uint MinRttUs;
+ internal uint BytesInFlight;
+ internal uint Cwnd;
+ internal uint SndWnd;
+ internal uint RcvWnd;
+ internal uint RcvBuf;
+ internal ulong BytesOut;
+ internal ulong BytesIn;
+ internal uint BytesReordered;
+ internal uint BytesRetrans;
+ internal uint FastRetrans;
+ internal uint DupAcksIn;
+ internal uint TimeoutEpisodes;
+ internal byte SynRetrans;
+ }
+ }
+}

src/libraries/System.Net.Http/ref/System.Net.Http.cs

@@ -353,6 +353,10 @@ public sealed partial class SocketsHttpHandler : System.Net.Http.HttpMessageHand
  {
  public SocketsHttpHandler() { }
  public static bool IsSupported { get { throw null; } }
+ public TimeSpan? FakeRtt { get { throw null; } set { } }
+ public int InitialStreamWindowSize { get { throw null; } set { } }
+ public int StreamWindowUpdateRatio { get { throw null; } set { } }
+ public int StreamWindowMagicMultiplier { get { throw null; } set { } }
  public bool AllowAutoRedirect { get { throw null; } set { } }
  public System.Net.DecompressionMethods AutomaticDecompression { get { throw null; } set { } }
  public System.TimeSpan ConnectTimeout { get { throw null; } set { } }

src/libraries/System.Net.Http/src/System.Net.Http.csproj

@@ -151,6 +151,7 @@
  <Compile Include="System\Net\Http\SocketsHttpHandler\Http2ProtocolException.cs" />
  <Compile Include="System\Net\Http\SocketsHttpHandler\Http2Stream.cs" />
  <Compile Include="System\Net\Http\SocketsHttpHandler\Http2StreamException.cs" />
+ <Compile Include="System\Net\Http\SocketsHttpHandler\Http2StreamWindowManager.cs" />
  <Compile Include="System\Net\Http\SocketsHttpHandler\Http3Connection.cs" />
  <Compile Include="System\Net\Http\SocketsHttpHandler\Http3ConnectionException.cs" />
  <Compile Include="System\Net\Http\SocketsHttpHandler\Http3ProtocolException.cs" />
@@ -411,6 +412,8 @@
  Link="Common\Interop\Windows\WinHttp\Interop.winhttp_types.cs" />
  <Compile Include="$(CommonPath)\Interop\Windows\WinHttp\Interop.winhttp.cs"
  Link="Common\Interop\Windows\WinHttp\Interop.winhttp.cs" />
+ <Compile Include="$(CommonPath)\Interop\Windows\WinSock\Interop.WSAIoctl.TcpInfo.cs"
+ Link="Common\Interop\Windows\WinSock\Interop.WSAIoctl.TcpInfo.cs" />
  <Compile Include="$(CommonPath)\System\CharArrayHelpers.cs"
  Link="Common\System\CharArrayHelpers.cs" />
  <Compile Include="$(CommonPath)\System\StringExtensions.cs"

src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/SocketsHttpHandler.cs

@@ -24,6 +24,30 @@ public bool UseCookies
  set => throw new PlatformNotSupportedException();
  }
 
+ public TimeSpan? FakeRtt
+ {
+ get => throw new PlatformNotSupportedException();
+ set => throw new PlatformNotSupportedException();
+ }
+
+ public int InitialStreamWindowSize
+ {
+ get => throw new PlatformNotSupportedException();
+ set => throw new PlatformNotSupportedException();
+ }
+
+ public int StreamWindowUpdateRatio
+ {
+ get => throw new PlatformNotSupportedException();
+ set => throw new PlatformNotSupportedException();
+ }
+
+ public int StreamWindowMagicMultiplier
+ {
+ get => throw new PlatformNotSupportedException();
+ set => throw new PlatformNotSupportedException();
+ }
+
  [AllowNull]
  public CookieContainer CookieContainer
  {

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs

@@ -38,6 +38,7 @@ internal sealed partial class Http2Connection : HttpConnectionBase, IDisposable
 
  private readonly CreditManager _connectionWindow;
  private readonly CreditManager _concurrentStreams;
+ private readonly RttEstimator? _rttEstimator;
 
  private int _nextStream;
  private bool _expectingSettingsAck;
@@ -79,21 +80,26 @@ internal sealed partial class Http2Connection : HttpConnectionBase, IDisposable
 #else
  private const int InitialConnectionBufferSize = 4096;
 #endif
-
- private const int DefaultInitialWindowSize = 65535;
+ // According to https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.2:
+ // "The connection flow-control window can only be changed using WINDOW_UPDATE frames."
+ // We need to initialize _connectionWindow with the value 65535, since
+ // higher values may violate the protocol, and lead to overflows,
+ // when the server (or an intermediate proxy) sens a very high connection WINDOW_UPDATE increment.
+ private const int DefaultInitialConnectionWindowSize = 65535;
 
  // We don't really care about limiting control flow at the connection level.
  // We limit it per stream, and the user controls how many streams are created.
  // So set the connection window size to a large value.
  private const int ConnectionWindowSize = 64 * 1024 * 1024;
+ private const int ConnectionWindowUpdateRatio = 8;
 
  // We hold off on sending WINDOW_UPDATE until we hit thi minimum threshold.
  // This value is somewhat arbitrary; the intent is to ensure it is much smaller than
  // the window size itself, or we risk stalling the server because it runs out of window space.
  // If we want to further reduce the frequency of WINDOW_UPDATEs, it's probably better to
  // increase the window size (and thus increase the threshold proportionally)
  // rather than just increase the threshold.
- private const int ConnectionWindowThreshold = ConnectionWindowSize / 8;
+ private const int ConnectionWindowThreshold = ConnectionWindowSize / ConnectionWindowUpdateRatio;
 
  // When buffering outgoing writes, we will automatically buffer up to this number of bytes.
  // Single writes that are larger than the buffer can cause the buffer to expand beyond
@@ -117,7 +123,7 @@ internal enum KeepAliveState
  private long _keepAlivePingTimeoutTimestamp;
  private volatile KeepAliveState _keepAliveState;
 
- public Http2Connection(HttpConnectionPool pool, Stream stream)
+ public Http2Connection(HttpConnectionPool pool, Stream stream, System.Net.Sockets.Socket? socket)
  {
  _pool = pool;
  _stream = stream;
@@ -128,13 +134,17 @@ public Http2Connection(HttpConnectionPool pool, Stream stream)
 
  _httpStreams = new Dictionary<int, Http2Stream>();
 
- _connectionWindow = new CreditManager(this, nameof(_connectionWindow), DefaultInitialWindowSize);
+ _connectionWindow = new CreditManager(this, nameof(_connectionWindow), DefaultInitialConnectionWindowSize);
  _concurrentStreams = new CreditManager(this, nameof(_concurrentStreams), InitialMaxConcurrentStreams);
+ InitialStreamWindowSize = pool.Settings._initialStreamWindowSize;
+ _rttEstimator = _pool.Settings._fakeRtt != null || socket != null ?
+ new RttEstimator(this, _pool.Settings._fakeRtt, socket) :
+ null;
 
  _writeChannel = Channel.CreateUnbounded<WriteQueueEntry>(s_channelOptions);
 
  _nextStream = 1;
- _initialWindowSize = DefaultInitialWindowSize;
+ _initialWindowSize = DefaultInitialConnectionWindowSize;
 
  _maxConcurrentStreams = InitialMaxConcurrentStreams;
  _pendingWindowUpdate = 0;
@@ -178,18 +188,24 @@ public async ValueTask SetupAsync()
  s_http2ConnectionPreface.AsSpan().CopyTo(_outgoingBuffer.AvailableSpan);
  _outgoingBuffer.Commit(s_http2ConnectionPreface.Length);
 
- // Send SETTINGS frame. Disable push promise.
- FrameHeader.WriteTo(_outgoingBuffer.AvailableSpan, FrameHeader.SettingLength, FrameType.Settings, FrameFlags.None, streamId: 0);
+ // Send SETTINGS frame. Disable push promise & set initial window size.
+ FrameHeader.WriteTo(_outgoingBuffer.AvailableSpan, 2*FrameHeader.SettingLength, FrameType.Settings, FrameFlags.None, streamId: 0);
  _outgoingBuffer.Commit(FrameHeader.Size);
  BinaryPrimitives.WriteUInt16BigEndian(_outgoingBuffer.AvailableSpan, (ushort)SettingId.EnablePush);
  _outgoingBuffer.Commit(2);
  BinaryPrimitives.WriteUInt32BigEndian(_outgoingBuffer.AvailableSpan, 0);
  _outgoingBuffer.Commit(4);
+ BinaryPrimitives.WriteUInt16BigEndian(_outgoingBuffer.AvailableSpan, (ushort)SettingId.InitialWindowSize);
+ _outgoingBuffer.Commit(2);
+ BinaryPrimitives.WriteUInt32BigEndian(_outgoingBuffer.AvailableSpan, (uint)InitialStreamWindowSize);
+ _outgoingBuffer.Commit(4);
 
  // Send initial connection-level WINDOW_UPDATE
+ uint windowUpdateAmount = (uint)(ConnectionWindowSize - InitialStreamWindowSize);
+ if (NetEventSource.Log.IsEnabled()) Trace($"Initial connection-level WINDOW_UPDATE, windowUpdateAmount={windowUpdateAmount}");
  FrameHeader.WriteTo(_outgoingBuffer.AvailableSpan, FrameHeader.WindowUpdateLength, FrameType.WindowUpdate, FrameFlags.None, streamId: 0);
  _outgoingBuffer.Commit(FrameHeader.Size);
- BinaryPrimitives.WriteUInt32BigEndian(_outgoingBuffer.AvailableSpan, ConnectionWindowSize - DefaultInitialWindowSize);
+ BinaryPrimitives.WriteUInt32BigEndian(_outgoingBuffer.AvailableSpan, windowUpdateAmount);
  _outgoingBuffer.Commit(4);
 
  await _stream.WriteAsync(_outgoingBuffer.ActiveMemory).ConfigureAwait(false);
@@ -574,6 +590,7 @@ private void ProcessDataFrame(FrameHeader frameHeader)
 
  if (frameData.Length > 0)
  {
+ _rttEstimator?.UpdateEstimation();
  ExtendWindow(frameData.Length);
  }
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs

@@ -36,7 +36,7 @@ private sealed class Http2Stream : IValueTaskSource, IHttpHeadersHandler, IHttpT
  private HttpResponseHeaders? _trailers;
 
  private MultiArrayBuffer _responseBuffer; // mutable struct, do not make this readonly
- private int _pendingWindowUpdate;
+ private Http2StreamWindowManager _windowManager;
  private CreditWaiter? _creditWaiter;
  private int _availableCredit;
  private readonly object _creditSyncObject = new object(); // split from SyncObject to avoid lock ordering problems with Http2Connection.SyncObject
@@ -85,11 +85,6 @@ private sealed class Http2Stream : IValueTaskSource, IHttpHeadersHandler, IHttpT
 
  private int _headerBudgetRemaining;
 
- private const int StreamWindowSize = DefaultInitialWindowSize;
-
- // See comment on ConnectionWindowThreshold.
- private const int StreamWindowThreshold = StreamWindowSize / 8;
-
  public Http2Stream(HttpRequestMessage request, Http2Connection connection)
  {
  _request = request;
@@ -102,7 +97,12 @@ public Http2Stream(HttpRequestMessage request, Http2Connection connection)
 
  _responseBuffer = new MultiArrayBuffer(InitialStreamBufferSize);
 
- _pendingWindowUpdate = 0;
+ _windowManager = connection._rttEstimator != null ?
+ new DynamicHttp2StreamWindowManager(connection, this) :
+ new Http2StreamWindowManager(connection, this);
+
+ Trace($"_windowManager: {_windowManager.GetType().Name}");
+
  _headerBudgetRemaining = connection._pool.Settings._maxResponseHeadersLength * 1024;
 
  if (_request.Content == null)
@@ -149,6 +149,8 @@ public void Initialize(int streamId, int initialWindowSize)
 
  public bool SendRequestFinished => _requestCompletionState != StreamCompletionState.InProgress;
 
+ public bool ExpectResponseData => _responseProtocolState == ResponseProtocolState.ExpectingData;
+
  public HttpResponseMessage GetAndClearResponse()
  {
  // Once SendAsync completes, the Http2Stream should no longer hold onto the response message.
@@ -805,7 +807,7 @@ public void OnResponseData(ReadOnlySpan<byte> buffer, bool endStream)
  break;
  }
 
- if (_responseBuffer.ActiveMemory.Length + buffer.Length > StreamWindowSize)
+ if (_responseBuffer.ActiveMemory.Length + buffer.Length > _windowManager.StreamWindowSize)
  {
  // Window size exceeded.
  ThrowProtocolError(Http2ProtocolErrorCode.FlowControlError);
@@ -1013,30 +1015,6 @@ public async Task ReadResponseHeadersAsync(CancellationToken cancellationToken)
  }
  }
 
- private void ExtendWindow(int amount)
- {
- Debug.Assert(amount > 0);
- Debug.Assert(_pendingWindowUpdate < StreamWindowThreshold);
-
- if (_responseProtocolState != ResponseProtocolState.ExpectingData)
- {
- // We are not expecting any more data (because we've either completed or aborted).
- // So no need to send any more WINDOW_UPDATEs.
- return;
- }
-
- _pendingWindowUpdate += amount;
- if (_pendingWindowUpdate < StreamWindowThreshold)
- {
- return;
- }
-
- int windowUpdateSize = _pendingWindowUpdate;
- _pendingWindowUpdate = 0;
-
- _connection.LogExceptions(_connection.SendWindowUpdateAsync(StreamId, windowUpdateSize));
- }
-
  private (bool wait, int bytesRead) TryReadFromBuffer(Span<byte> buffer, bool partOfSyncRead = false)
  {
  Debug.Assert(buffer.Length > 0);
@@ -1089,7 +1067,7 @@ public int ReadData(Span<byte> buffer, HttpResponseMessage responseMessage)
 
  if (bytesRead != 0)
  {
- ExtendWindow(bytesRead);
+ _windowManager.AdjustWindow(bytesRead);
  }
  else
  {
@@ -1118,7 +1096,7 @@ public async ValueTask<int> ReadDataAsync(Memory<byte> buffer, HttpResponseMessa
 
  if (bytesRead != 0)
  {
- ExtendWindow(bytesRead);
+ _windowManager.AdjustWindow(bytesRead);
  }
  else
  {
@@ -1148,7 +1126,7 @@ public void CopyTo(HttpResponseMessage responseMessage, Stream destination, int
 
  if (bytesRead != 0)
  {
- ExtendWindow(bytesRead);
+ _windowManager.AdjustWindow(bytesRead);
  destination.Write(new ReadOnlySpan<byte>(buffer, 0, bytesRead));
  }
  else
@@ -1184,7 +1162,7 @@ public async Task CopyToAsync(HttpResponseMessage responseMessage, Stream destin
 
  if (bytesRead != 0)
  {
- ExtendWindow(bytesRead);
+ _windowManager.AdjustWindow(bytesRead);
  await destination.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, bytesRead), cancellationToken).ConfigureAwait(false);
  }
  else