Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[HTTP/3] Stress hack for msquic dropping connections #84793

Merged
merged 13 commits into from
Apr 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions eng/docker/build-docker-sdk.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ if ($buildWindowsContainers)
# 2. Runtime pack (microsoft.netcore.app.runtime.win-x64)
# 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
# 4. testhost
# 5. msquic interop sources (needed for HttpStress)
$binArtifacts = "$REPO_ROOT_DIR\artifacts\bin"
$dockerContext = "$REPO_ROOT_DIR\artifacts\docker-context"

Expand All @@ -51,6 +52,8 @@ if ($buildWindowsContainers)
-Destination $dockerContext\testhost
Copy-Item -Recurse -Path $REPO_ROOT_DIR\eng\targetingpacks.targets `
-Destination $dockerContext\targetingpacks.targets
Copy-Item -Recurse -Path $REPO_ROOT_DIR\src\libraries\System.Net.Quic\src\System\Net\Quic\Interop `
-Destination $dockerContext\msquic-interop

# In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress):
$testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
Expand Down
5 changes: 5 additions & 0 deletions eng/docker/libraries-sdk.linux.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ RUN bash ./dotnet-install.sh --channel $_DOTNET_INSTALL_CHANNEL --quality daily
# 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
# 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
# 4. testhost
# 5. msquic interop sources (needed for HttpStress)

COPY --from=corefxbuild \
/repo/artifacts/bin/microsoft.netcore.app.ref \
Expand All @@ -43,6 +44,10 @@ COPY --from=corefxbuild \
/repo/artifacts/bin/testhost \
/live-runtime-artifacts/testhost

COPY --from=corefxbuild \
/repo/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop \
/live-runtime-artifacts/msquic-interop

# Add AspNetCore bits to testhost:
ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
ENV _ASPNETCORE_DEST="/live-runtime-artifacts/testhost/net$VERSION-linux-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@

<!-- Stress projects have their own global.json, the directory above that also has it is the repository root. -->
<RepositoryRoot>$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)../, global.json))/</RepositoryRoot>


<MsQuicInteropIncludes Condition="'$(MsQuicInteropIncludes)' == ''">$(RepositoryRoot)src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/*.cs</MsQuicInteropIncludes>
<TargetingPacksTargetsLocation Condition="'$(TargetingPacksTargetsLocation)' == ''">$(RepositoryRoot)eng/targetingpacks.targets</TargetingPacksTargetsLocation>
<ProductVersion>8.0.0</ProductVersion>
<NetCoreAppCurrent>net8.0</NetCoreAppCurrent>
Expand All @@ -14,4 +15,4 @@
<MicrosoftNetCoreAppRefPackDir Condition="'$(MicrosoftNetCoreAppRefPackDir)' == ''" >$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.ref/</MicrosoftNetCoreAppRefPackDir>
<MicrosoftNetCoreAppRuntimePackDir Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == ''">$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.runtime.$(OutputRID)/$(Configuration)/</MicrosoftNetCoreAppRuntimePackDir>
</PropertyGroup>
</Project>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ WORKDIR /app
COPY . .

RUN dotnet build -c $CONFIGURATION \
-p:MsQuicInteropIncludes="/live-runtime-artifacts/msquic-interop/*.cs" \
-p:TargetingPacksTargetsLocation=/live-runtime-artifacts/targetingpacks.targets \
-p:MicrosoftNetCoreAppRefPackDir=/live-runtime-artifacts/microsoft.netcore.app.ref/ \
-p:MicrosoftNetCoreAppRuntimePackDir=/live-runtime-artifacts/microsoft.netcore.app.runtime.linux-x64/$CONFIGURATION/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
<TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
<Nullable>enable</Nullable>
<EnablePreviewFeatures>True</EnablePreviewFeatures>
<NoWarn>CA2252</NoWarn>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

<ItemGroup>
Expand All @@ -15,6 +17,11 @@
<PackageReference Include="System.Net.Http.WinHttpHandler" Version="4.5.4" />
</ItemGroup>

<!-- Shared production code -->
<ItemGroup>
<Compile Include="$(MsQuicInteropIncludes)" LinkBase="MsQuicInterop" />
</ItemGroup>

<PropertyGroup>
<!-- These may lead to duplicate generated classes with local (non-docker) Linux builds. -->
<GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
using System.Threading.Tasks;
using System.Net;
using HttpStress;
using System.Net.Quic;
using Microsoft.Quic;

[assembly:SupportedOSPlatform("windows")]
[assembly:SupportedOSPlatform("linux")]
Expand All @@ -26,6 +28,8 @@ public static class Program
{
public enum ExitCode { Success = 0, StressError = 1, CliError = 2 };

public static readonly bool IsQuicSupported = QuicListener.IsSupported && QuicConnection.IsSupported;

public static async Task<int> Main(string[] args)
{
if (!TryParseCli(args, out Configuration? config))
Expand Down Expand Up @@ -158,6 +162,9 @@ private static async Task<ExitCode> Run(Configuration config)

string GetAssemblyInfo(Assembly assembly) => $"{assembly.Location}, modified {new FileInfo(assembly.Location).LastWriteTime}";

Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi");
string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ManickaP unfortunately, this line fails with NullReferenceException in release builds. I assume the MsQuicLibraryVersion is being trimmed away, since it's unused in the production code.

We could workaround this by moving the version extraction to a shared test-utility (together with the MaxWorkerQueue hack helper), but IMHO it's not worth it unless we think that it's super-important to know the msquic version for stress test troubleshooting.

For now I'm just removing it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it possible that there is image without MsQuic?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you mean? If you are suspecting lack of msquic as the root cause of this error - it's not the case. Removing code messing with MsQuicLibraryVersion makes things work (= HTTP/3 stress runs).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it possible that there is image without MsQuic?

In that case you'd get "unknown" as the MsQuic version, the code expects that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another option would be to tell the trimmer to keep it there via DynamicDependency. And this is just one string.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should try to clarify whether test-only usage of DynamicDependency is ok, without that, I prefer to defensively assume it's not.


Console.WriteLine(" .NET Core: " + GetAssemblyInfo(typeof(object).Assembly));
Console.WriteLine(" ASP.NET Core: " + GetAssemblyInfo(typeof(WebHost).Assembly));
Console.WriteLine(" System.Net.Http: " + GetAssemblyInfo(typeof(System.Net.Http.HttpClient).Assembly));
Expand All @@ -169,6 +176,8 @@ private static async Task<ExitCode> Run(Configuration config)
Console.WriteLine(" Concurrency: " + config.ConcurrentRequests);
Console.WriteLine(" Content Length: " + config.MaxContentLength);
Console.WriteLine(" HTTP Version: " + config.HttpVersion);
Console.WriteLine(" QUIC supported: " + (IsQuicSupported ? "yes" : "no"));
Console.WriteLine(" MsQuic Version: " + msQuicLibraryVersion);
Console.WriteLine(" Lifetime: " + (config.ConnectionLifetime.HasValue ? $"{config.ConnectionLifetime.Value.TotalMilliseconds}ms" : "(infinite)"));
Console.WriteLine(" Operations: " + string.Join(", ", usedClientOperations.Select(o => o.name)));
Console.WriteLine(" Random Seed: " + config.RandomSeed);
Expand All @@ -177,6 +186,23 @@ private static async Task<ExitCode> Run(Configuration config)
Console.WriteLine("Query Parameters: " + config.MaxParameters);
Console.WriteLine();

if (config.HttpVersion == HttpVersion.Version30 && IsQuicSupported)
{
unsafe
{
// If the system gets overloaded, MsQuic has a tendency to drop incoming connections, see https://github.com/dotnet/runtime/issues/55979.
// So in case we're running H/3 stress test, we're using the same hack as for System.Net.Quic tests, which increases the time limit for pending operations in MsQuic thread pool.
object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty<object?>())));
QUIC_SETTINGS settings = default(QUIC_SETTINGS);
settings.IsSet.MaxWorkerQueueDelayUs = 1;
settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default
if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
{
Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs.");
}
}
}

StressServer? server = null;
if (config.RunMode.HasFlag(RunMode.server))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ if (-not ([string]::IsNullOrEmpty($args[0]))) {

$LibrariesConfiguration = "Release"
if (-not ([string]::IsNullOrEmpty($args[1]))) {
$LibrariesConfiguration = $args[0]
$LibrariesConfiguration = $args[1]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixes in this file are unrelated. Both $StressConfiguration and $LibrariesConfiguration referred to $args[0] 🤦 .

}

$TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64"
Expand Down Expand Up @@ -53,11 +53,11 @@ if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) {
Write-Host "Building solution."
dotnet build -c $StressConfiguration

$Runscript=".\run-stress-$LibrariesConfiguration-$StressConfiguration.ps1"
$Runscript=".\run-stress-$StressConfiguration-$LibrariesConfiguration.ps1"
if (-not (Test-Path $Runscript)) {
Write-Host "Generating Runscript."
Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec --roll-forward Major ./bin/$StressConfiguration/net$Version/HttpStress.dll `$args"
}

Write-Host "To run tests type:"
Write-Host "$Runscript [stress test args]"
Write-Host "$Runscript [stress test args]"
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ ARG VERSION=8.0
ARG CONFIGURATION=Release

RUN dotnet build -c $env:CONFIGURATION `
-p:MsQuicInteropIncludes="C:/live-runtime-artifacts/msquic-interop/*.cs" `
-p:TargetingPacksTargetsLocation=C:/live-runtime-artifacts/targetingpacks.targets `
-p:MicrosoftNetCoreAppRefPackDir=C:/live-runtime-artifacts/microsoft.netcore.app.ref/ `
-p:MicrosoftNetCoreAppRuntimePackDir=C:/live-runtime-artifacts/microsoft.netcore.app.runtime.win-x64/$env:CONFIGURATION/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ static MsQuicApi()
}
string? gitHash = Marshal.PtrToStringUTF8((IntPtr)libGitHash);

MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} version={version} commit={gitHash}";
MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} {version} ({gitHash})";

if (version < s_minMsQuicVersion)
{
Expand All @@ -143,7 +143,7 @@ static MsQuicApi()

if (NetEventSource.Log.IsEnabled())
{
NetEventSource.Info(null, $"Loaded MsQuic library version '{version}', commit '{gitHash}'.");
NetEventSource.Info(null, $"Loaded MsQuic library '{MsQuicLibraryVersion}'.");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Brilliant 😆

}

// Assume SChannel is being used on windows and query for the actual provider from the library if querying is supported
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ public async Task ConnectWithServerCertificateCallback()
// TODO: the exception may change if we implement https://github.com/dotnet/runtime/issues/73152 to make server close
// connections with CONNECTION_REFUSED in such cases
var authEx = await Assert.ThrowsAsync<AuthenticationException>(() => clientTask);
Assert.Contains("UserCanceled", authEx.Message);
Assert.Contains(TlsAlertMessage.UserCanceled.ToString(), authEx.Message);
Assert.Equal(clientOptions.ClientAuthenticationOptions.TargetHost, receivedHostName);
await Assert.ThrowsAsync<ArgumentException>(async () => await listener.AcceptConnectionAsync());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
using Xunit.Abstractions;
using System.Diagnostics.Tracing;
using System.Net.Sockets;
using System.Reflection;
using Microsoft.Quic;
using static Microsoft.Quic.MsQuic;

namespace System.Net.Quic.Tests
{
Expand Down Expand Up @@ -44,14 +44,20 @@ public abstract class QuicTestBase : IDisposable

static unsafe QuicTestBase()
{
Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{MsQuicApi.MsQuicLibraryVersion}'.");
// If any of the reflection bellow breaks due to changes in "System.Net.Quic.MsQuicApi", also check and fix HttpStress project as it uses the same hack.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if this should go somewhere to Common/test/.... so we can avoid duplication.
It may be worth of adding explanation as this is unusual craft.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, I'll look into it.

Copy link
Member

@antonfirsov antonfirsov Apr 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should keep in mind that whatever utility helper we add, its' file has to be also copied to the container, so avoiding duplication has a higher cost in complexity here than normally. At the moment I'm undecided if it's worth it, would prefer to do the docker copying work mentioned in #84793 (comment) first, planning to jump to it tomorrow.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I'm holding back. Let me know when/if I should look into it.

Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi");

string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{msQuicLibraryVersion}'.");

if (IsSupported)
{
object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty<object?>())));
QUIC_SETTINGS settings = default(QUIC_SETTINGS);
settings.IsSet.MaxWorkerQueueDelayUs = 1;
settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default
if (StatusFailed(MsQuicApi.Api.ApiTable->SetParam(null, QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
{
Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs.");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
<Compile Include="$(CommonPath)System\Net\ArrayBuffer.cs" Link="ProductionCode\Common\System\Net\ArrayBuffer.cs" />
<Compile Include="$(CommonPath)System\Net\MultiArrayBuffer.cs" Link="ProductionCode\Common\System\Net\MultiArrayBuffer.cs" />
<Compile Include="$(CommonPath)System\Net\StreamBuffer.cs" Link="ProductionCode\Common\System\Net\StreamBuffer.cs" />
<Compile Include="$(CommonPath)System\Net\Security\TlsAlertMessage.cs" Link="Common\System\Net\Security\TlsAlertMessage.cs" />
<Compile Include="$(CommonTestPath)System\IO\ConnectedStreams.cs" Link="Common\System\IO\ConnectedStreams.cs" />
<Compile Include="$(CommonTestPath)System\Net\Capability.Security.cs" Link="Common\System\Net\Capability.Security.cs" />
<Compile Include="$(CommonTestPath)System\Net\Configuration.cs" Link="Common\System\Net\Configuration.cs" />
Expand All @@ -31,18 +32,8 @@
</ItemGroup>
<!-- Shared production code -->
<ItemGroup>
<Compile Include="$(CommonPath)System\Net\Logging\NetEventSource.Common.cs" Link="Common\System\Net\Logging\NetEventSource.Common.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicApi.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicApi.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicApi.NativeMethods.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicApi.NativeMethods.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicSafeHandle.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicSafeHandle.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\ThrowHelper.cs" Link="ProductionCode\System\Net\Quic\Internal\ThrowHelper.cs" />
<Compile Include="..\..\src\System\Net\Quic\Interop\*.cs" Link="ProductionCode\System\Net\Quic\Interop\*.cs" />
<Compile Include="..\..\src\System\Net\Quic\NetEventSource.Quic.cs" Link="ProductionCode\System\Net\Quic\NetEventSource.Quic.cs" />
<Compile Include="..\..\src\System\Net\Quic\QuicDefaults.cs" Link="ProductionCode\System\Net\Quic\QuicDefaults.cs" />
<Compile Include="$(CommonPath)Interop\Windows\Interop.Libraries.cs" Link="Common\Interop\Windows\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'windows'" />
<Compile Include="$(CommonPath)Interop\Linux\Interop.Libraries.cs" Link="Common\Interop\Linux\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'linux'" />
<Compile Include="$(CommonPath)Interop\OSX\Interop.Libraries.cs" Link="Common\Interop\OSX\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'osx'" />
<Compile Include="$(CommonPath)System\Net\Security\TlsAlertMessage.cs" Link="Common\System\Net\Security\TlsAlertMessage.cs" />
<Compile Include="..\..\src\System\Net\Quic\Interop\*.cs" LinkBase="ProductionCode\System\Net\Quic\Interop" />
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I changed this to use LinkBase, Visual Studio is going crazy when Link is being used with wildcards:

image

</ItemGroup>
<ItemGroup>
<ProjectReference Include="$(CommonTestPath)StreamConformanceTests\StreamConformanceTests.csproj" />
Expand Down