diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 892e3f6cd9..c3174a5dd7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -182,6 +182,8 @@ jobs: fail-fast: false matrix: vec: [ + { config: "Debug", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test", xdp: "-UseXdp" }, + { config: "Release", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test", xdp: "-UseXdp" }, { config: "Debug", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" }, { config: "Release", plat: "winkernel", os: "windows-2022", arch: "x64", tls: "schannel", build: "-Test" }, { config: "Debug", plat: "winkernel", os: "WinServerPrerelease", arch: "x64", tls: "schannel", build: "-Test" }, @@ -205,7 +207,7 @@ jobs: path: artifacts - name: Prepare Machine shell: pwsh - run: scripts/prepare-machine.ps1 -ForTest -ForKernel + run: scripts/prepare-machine.ps1 -ForTest -ForKernel ${{ matrix.vec.xdp }} - name: Install ETW Manifest shell: pwsh run: | @@ -216,12 +218,12 @@ jobs: - name: Test shell: pwsh timeout-minutes: 90 - run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -OsRunner ${{ matrix.vec.os }} -GHA -LogProfile Full.Light -GenerateXmlResults -Kernel -Filter -*ValidateConfiguration:*ValidAlpnLengths:*ResumeRejection*:*ClientCertificate*:*LoadBalanced* + run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -OsRunner ${{ matrix.vec.os }} ${{ matrix.vec.xdp }} -GHA -LogProfile Full.Light -GenerateXmlResults -Kernel -Filter -*ValidateConfiguration:*ValidAlpnLengths:*ResumeRejection*:*ClientCertificate*:*LoadBalanced* - name: Upload on Failure uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 if: failure() with: - name: BVT-Kernel-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }} + name: BVT-Kernel-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }} path: artifacts interop-winlatest: diff --git a/scripts/clog.inputs b/scripts/clog.inputs index 0b3a68f323..da956173f1 100644 --- a/scripts/clog.inputs +++ b/scripts/clog.inputs @@ -18,9 +18,12 @@ ../src/platform/datapath_xplat.c ../src/platform/datapath_raw_dpdk.c ../src/platform/datapath_raw_socket.c +../src/platform/datapath_raw_socket_common.c ../src/platform/datapath_raw_socket_win.c +../src/platform/datapath_raw_socket_winuser.c ../src/platform/datapath_raw_socket_linux.c -../src/platform/datapath_raw_xdp_win.c +../src/platform/datapath_raw_xdp_winuser.c +../src/platform/datapath_raw_xdp_winkernel.c ../src/platform/datapath_raw_xdp_linux.c ../src/platform/datapath_raw_win.c ../src/platform/datapath_raw_linux.c diff --git a/scripts/prepare-machine.ps1 b/scripts/prepare-machine.ps1 index 318648076d..5c79e5133d 100644 --- a/scripts/prepare-machine.ps1 +++ b/scripts/prepare-machine.ps1 @@ -230,9 +230,17 @@ function Install-SigningCertificates { # NB: XDP can be uninstalled via Uninstall-Xdp function Install-Xdp-Driver { if (!$IsWindows) { return } # Windows only + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + Write-Host "Downloading XDP msi" $MsiPath = Join-Path $ArtifactsPath "xdp.msi" Invoke-WebRequest -Uri (Get-Content (Join-Path $PSScriptRoot "xdp.json") | ConvertFrom-Json).installer -OutFile $MsiPath + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $MsiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' Write-Host "Installing XDP driver" msiexec.exe /i $MsiPath /quiet | Out-Null } diff --git a/scripts/quic_callback.ps1 b/scripts/quic_callback.ps1 index 9fc6f27d69..2acde4eff3 100644 --- a/scripts/quic_callback.ps1 +++ b/scripts/quic_callback.ps1 @@ -78,10 +78,18 @@ if ($Command.Contains("/home/secnetperf/_work/quic/artifacts/bin/linux/x64_Relea ./artifacts/bin/windows/x64_Release_schannel/secnetperf -exec:$mode -io:$io -stats:$stats } elseif ($Command.Contains("Install_XDP")) { Write-Host "Executing command: Install_XDP" + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + Write-Host "(SERVER) Downloading XDP installer" $installerUri = $Command.Split(";")[1] $msiPath = Repo-Path "xdp.msi" Invoke-WebRequest -Uri $installerUri -OutFile $msiPath -UseBasicParsing + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $msiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' Write-Host "(SERVER) Installing XDP. Msi path: $msiPath" msiexec.exe /i $msiPath /quiet | Out-Host Wait-DriverStarted "xdp" 10000 diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index bff5adba31..3728724056 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -136,11 +136,19 @@ function Wait-DriverStarted { # Download and install XDP on both local and remote machines. function Install-XDP { param ($Session, $RemoteDir) + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + $installerUri = (Get-Content (Join-Path $PSScriptRoot "xdp.json") | ConvertFrom-Json).installer $msiPath = Repo-Path "artifacts/xdp.msi" Write-Host "Downloading XDP installer" whoami Invoke-WebRequest -Uri $installerUri -OutFile $msiPath -UseBasicParsing + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $msiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' Write-Host "Installing XDP driver locally" msiexec.exe /i $msiPath /quiet | Out-Null $Size = Get-FileHash $msiPath @@ -158,6 +166,14 @@ function Install-XDP { Copy-Item -ToSession $Session $msiPath -Destination $remoteMsiPath $WaitDriverStartedStr = "${function:Wait-DriverStarted}" Invoke-Command -Session $Session -ScriptBlock { + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $Using:remoteMsiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' msiexec.exe /i $Using:remoteMsiPath /quiet | Out-Host $WaitDriverStarted = [scriptblock]::Create($Using:WaitDriverStartedStr) & $WaitDriverStarted xdp 10000 diff --git a/src/bin/winkernel/msquic.kernel.vcxproj b/src/bin/winkernel/msquic.kernel.vcxproj index 1fe8ed84fc..4920d081f3 100644 --- a/src/bin/winkernel/msquic.kernel.vcxproj +++ b/src/bin/winkernel/msquic.kernel.vcxproj @@ -92,7 +92,12 @@ - ..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir);%(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 @@ -105,7 +110,12 @@ UseLinkTimeCodeGeneration - ..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(UM_IncludePath);%(AdditionalIncludeDirectories) + + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(UM_IncludePath);%(AdditionalIncludeDirectories) + diff --git a/src/bin/winkernel/msquicpriv.kernel.vcxproj b/src/bin/winkernel/msquicpriv.kernel.vcxproj index c8983b8c56..ae829162b6 100644 --- a/src/bin/winkernel/msquicpriv.kernel.vcxproj +++ b/src/bin/winkernel/msquicpriv.kernel.vcxproj @@ -92,7 +92,12 @@ - ..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir);%(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 @@ -105,7 +110,13 @@ UseLinkTimeCodeGeneration - ..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(UM_IncludePath);%(AdditionalIncludeDirectories) + + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(UM_IncludePath); + %(AdditionalIncludeDirectories) + diff --git a/src/core/core.kernel.vcxproj b/src/core/core.kernel.vcxproj index 798ce03a3e..4d256aacdb 100644 --- a/src/core/core.kernel.vcxproj +++ b/src/core/core.kernel.vcxproj @@ -148,7 +148,12 @@ - ..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir);%(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/core/worker.c b/src/core/worker.c index 6338834413..db9977a91f 100644 --- a/src/core/worker.c +++ b/src/core/worker.c @@ -96,13 +96,10 @@ QuicWorkerInitialize( Worker->ExecutionContext.NextTimeUs = UINT64_MAX; Worker->ExecutionContext.Ready = TRUE; -#ifndef _KERNEL_MODE // Not supported on kernel mode if (ExecProfile != QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT) { Worker->IsExternal = TRUE; CxPlatAddExecutionContext(&MsQuicLib.WorkerPool, &Worker->ExecutionContext, PartitionIndex); - } else -#endif // _KERNEL_MODE - { + } else { uint16_t ThreadFlags; switch (ExecProfile) { default: diff --git a/src/generated/linux/datapath_raw_socket.c.clog.h b/src/generated/linux/datapath_raw_socket.c.clog.h index 182db30f32..763d577d44 100644 --- a/src/generated/linux/datapath_raw_socket.c.clog.h +++ b/src/generated/linux/datapath_raw_socket.c.clog.h @@ -14,10 +14,6 @@ #include "datapath_raw_socket.c.clog.h.lttng.h" #endif #include -#ifndef _clog_MACRO_QuicTraceLogVerbose -#define _clog_MACRO_QuicTraceLogVerbose 1 -#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) -#endif #ifndef _clog_MACRO_QuicTraceLogConnInfo #define _clog_MACRO_QuicTraceLogConnInfo 1 #define QuicTraceLogConnInfo(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) @@ -29,26 +25,6 @@ #ifdef __cplusplus extern "C" { #endif -/*---------------------------------------------------------- -// Decoder Ring for DatapathTcpAuxBinding -// [data][%p] Binding TCP socket to %s -// QuicTraceLogVerbose( - DatapathTcpAuxBinding, - "[data][%p] Binding TCP socket to %s", - Socket, - LocalAddressString.Address); -// arg2 = arg2 = Socket = arg2 -// arg3 = arg3 = LocalAddressString.Address = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_DatapathTcpAuxBinding -#define _clog_4_ARGS_TRACE_DatapathTcpAuxBinding(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_SOCKET_C, DatapathTcpAuxBinding , arg2, arg3);\ - -#endif - - - - /*---------------------------------------------------------- // Decoder Ring for RouteResolutionEnd // [conn][%p] Route resolution completed on Path[%hhu] with L2 address %hhx:%hhx:%hhx:%hhx:%hhx:%hhx @@ -87,12 +63,12 @@ tracepoint(CLOG_DATAPATH_RAW_SOCKET_C, RouteResolutionEnd , arg1, arg3, arg4, ar // QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", - Socket, - Error, - "closesocket"); -// arg2 = arg2 = Socket = arg2 -// arg3 = arg3 = Error = arg3 -// arg4 = arg4 = "closesocket" = arg4 + Datapath, + Length, + "packet is too small for a UDP header"); +// arg2 = arg2 = Datapath = arg2 +// arg3 = arg3 = Length = arg3 +// arg4 = arg4 = "packet is too small for a UDP header" = arg4 ----------------------------------------------------------*/ #ifndef _clog_5_ARGS_TRACE_DatapathErrorStatus #define _clog_5_ARGS_TRACE_DatapathErrorStatus(uniqueId, encoded_arg_string, arg2, arg3, arg4)\ diff --git a/src/generated/linux/datapath_raw_socket.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_socket.c.clog.h.lttng.h index 66f2d44ad9..bc38c32230 100644 --- a/src/generated/linux/datapath_raw_socket.c.clog.h.lttng.h +++ b/src/generated/linux/datapath_raw_socket.c.clog.h.lttng.h @@ -1,29 +1,6 @@ -/*---------------------------------------------------------- -// Decoder Ring for DatapathTcpAuxBinding -// [data][%p] Binding TCP socket to %s -// QuicTraceLogVerbose( - DatapathTcpAuxBinding, - "[data][%p] Binding TCP socket to %s", - Socket, - LocalAddressString.Address); -// arg2 = arg2 = Socket = arg2 -// arg3 = arg3 = LocalAddressString.Address = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_C, DatapathTcpAuxBinding, - TP_ARGS( - const void *, arg2, - const char *, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_string(arg3, arg3) - ) -) - - - /*---------------------------------------------------------- // Decoder Ring for RouteResolutionEnd // [conn][%p] Route resolution completed on Path[%hhu] with L2 address %hhx:%hhx:%hhx:%hhx:%hhx:%hhx @@ -77,12 +54,12 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_C, RouteResolutionEnd, // QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", - Socket, - Error, - "closesocket"); -// arg2 = arg2 = Socket = arg2 -// arg3 = arg3 = Error = arg3 -// arg4 = arg4 = "closesocket" = arg4 + Datapath, + Length, + "packet is too small for a UDP header"); +// arg2 = arg2 = Datapath = arg2 +// arg3 = arg3 = Length = arg3 +// arg4 = arg4 = "packet is too small for a UDP header" = arg4 ----------------------------------------------------------*/ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_C, DatapathErrorStatus, TP_ARGS( diff --git a/src/generated/linux/datapath_raw_socket_common.c.clog.h b/src/generated/linux/datapath_raw_socket_common.c.clog.h new file mode 100644 index 0000000000..88c23c974c --- /dev/null +++ b/src/generated/linux/datapath_raw_socket_common.c.clog.h @@ -0,0 +1,75 @@ +#ifndef CLOG_DO_NOT_INCLUDE_HEADER +#include +#endif +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER CLOG_DATAPATH_RAW_SOCKET_COMMON_C +#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#define TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "datapath_raw_socket_common.c.clog.h.lttng.h" +#if !defined(DEF_CLOG_DATAPATH_RAW_SOCKET_COMMON_C) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define DEF_CLOG_DATAPATH_RAW_SOCKET_COMMON_C +#include +#define __int64 __int64_t +#include "datapath_raw_socket_common.c.clog.h.lttng.h" +#endif +#include +#ifndef _clog_MACRO_QuicTraceLogVerbose +#define _clog_MACRO_QuicTraceLogVerbose 1 +#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifndef _clog_MACRO_QuicTraceEvent +#define _clog_MACRO_QuicTraceEvent 1 +#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifdef __cplusplus +extern "C" { +#endif +/*---------------------------------------------------------- +// Decoder Ring for DatapathTcpAuxBinding +// [data][%p] Binding TCP socket to %s +// QuicTraceLogVerbose( + DatapathTcpAuxBinding, + "[data][%p] Binding TCP socket to %s", + Socket, + LocalAddressString.Address); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = LocalAddressString.Address = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_DatapathTcpAuxBinding +#define _clog_4_ARGS_TRACE_DatapathTcpAuxBinding(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_COMMON_C, DatapathTcpAuxBinding , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathErrorStatus +// [data][%p] ERROR, %u, %s. +// QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Error, + "closesocket"); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = Error = arg3 +// arg4 = arg4 = "closesocket" = arg4 +----------------------------------------------------------*/ +#ifndef _clog_5_ARGS_TRACE_DatapathErrorStatus +#define _clog_5_ARGS_TRACE_DatapathErrorStatus(uniqueId, encoded_arg_string, arg2, arg3, arg4)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_COMMON_C, DatapathErrorStatus , arg2, arg3, arg4);\ + +#endif + + + + +#ifdef __cplusplus +} +#endif +#ifdef CLOG_INLINE_IMPLEMENTATION +#include "quic.clog_datapath_raw_socket_common.c.clog.h.c" +#endif diff --git a/src/generated/linux/datapath_raw_socket_common.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_socket_common.c.clog.h.lttng.h new file mode 100644 index 0000000000..97bc22a702 --- /dev/null +++ b/src/generated/linux/datapath_raw_socket_common.c.clog.h.lttng.h @@ -0,0 +1,50 @@ + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathTcpAuxBinding +// [data][%p] Binding TCP socket to %s +// QuicTraceLogVerbose( + DatapathTcpAuxBinding, + "[data][%p] Binding TCP socket to %s", + Socket, + LocalAddressString.Address); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = LocalAddressString.Address = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_COMMON_C, DatapathTcpAuxBinding, + TP_ARGS( + const void *, arg2, + const char *, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_string(arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathErrorStatus +// [data][%p] ERROR, %u, %s. +// QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Error, + "closesocket"); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = Error = arg3 +// arg4 = arg4 = "closesocket" = arg4 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_COMMON_C, DatapathErrorStatus, + TP_ARGS( + const void *, arg2, + unsigned int, arg3, + const char *, arg4), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ctf_string(arg4, arg4) + ) +) diff --git a/src/generated/linux/datapath_raw_socket_winuser.c.clog.h b/src/generated/linux/datapath_raw_socket_winuser.c.clog.h new file mode 100644 index 0000000000..436c4ea3ec --- /dev/null +++ b/src/generated/linux/datapath_raw_socket_winuser.c.clog.h @@ -0,0 +1,179 @@ +#ifndef CLOG_DO_NOT_INCLUDE_HEADER +#include +#endif +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER CLOG_DATAPATH_RAW_SOCKET_WINUSER_C +#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#define TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "datapath_raw_socket_winuser.c.clog.h.lttng.h" +#if !defined(DEF_CLOG_DATAPATH_RAW_SOCKET_WINUSER_C) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define DEF_CLOG_DATAPATH_RAW_SOCKET_WINUSER_C +#include +#define __int64 __int64_t +#include "datapath_raw_socket_winuser.c.clog.h.lttng.h" +#endif +#include +#ifndef _clog_MACRO_QuicTraceLogConnInfo +#define _clog_MACRO_QuicTraceLogConnInfo 1 +#define QuicTraceLogConnInfo(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifndef _clog_MACRO_QuicTraceEvent +#define _clog_MACRO_QuicTraceEvent 1 +#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifdef __cplusplus +extern "C" { +#endif +/*---------------------------------------------------------- +// Decoder Ring for RouteResolutionStart +// [conn][%p] Starting to look up neighbor on Path[%hhu] with status %u +// QuicTraceLogConnInfo( + RouteResolutionStart, + Context, + "Starting to look up neighbor on Path[%hhu] with status %u", + PathId, + Status); +// arg1 = arg1 = Context = arg1 +// arg3 = arg3 = PathId = arg3 +// arg4 = arg4 = Status = arg4 +----------------------------------------------------------*/ +#ifndef _clog_5_ARGS_TRACE_RouteResolutionStart +#define _clog_5_ARGS_TRACE_RouteResolutionStart(uniqueId, arg1, encoded_arg_string, arg3, arg4)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, RouteResolutionStart , arg1, arg3, arg4);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryErrorStatus +// [ lib] ERROR, %u, %s. +// QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + WsaError, + "WSAStartup"); +// arg2 = arg2 = WsaError = arg2 +// arg3 = arg3 = "WSAStartup" = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_LibraryErrorStatus +#define _clog_4_ARGS_TRACE_LibraryErrorStatus(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, LibraryErrorStatus , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathGetRouteStart +// [data][%p] Querying route, local=%!ADDR!, remote=%!ADDR! +// QuicTraceEvent( + DatapathGetRouteStart, + "[data][%p] Querying route, local=%!ADDR!, remote=%!ADDR!", + Socket, + CASTED_CLOG_BYTEARRAY(sizeof(Route->LocalAddress), &Route->LocalAddress), + CASTED_CLOG_BYTEARRAY(sizeof(Route->RemoteAddress), &Route->RemoteAddress)); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = CASTED_CLOG_BYTEARRAY(sizeof(Route->LocalAddress), &Route->LocalAddress) = arg3 +// arg4 = arg4 = CASTED_CLOG_BYTEARRAY(sizeof(Route->RemoteAddress), &Route->RemoteAddress) = arg4 +----------------------------------------------------------*/ +#ifndef _clog_7_ARGS_TRACE_DatapathGetRouteStart +#define _clog_7_ARGS_TRACE_DatapathGetRouteStart(uniqueId, encoded_arg_string, arg2, arg3, arg3_len, arg4, arg4_len)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathGetRouteStart , arg2, arg3_len, arg3, arg4_len, arg4);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathErrorStatus +// [data][%p] ERROR, %u, %s. +// QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Status, + "GetBestRoute2"); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = Status = arg3 +// arg4 = arg4 = "GetBestRoute2" = arg4 +----------------------------------------------------------*/ +#ifndef _clog_5_ARGS_TRACE_DatapathErrorStatus +#define _clog_5_ARGS_TRACE_DatapathErrorStatus(uniqueId, encoded_arg_string, arg2, arg3, arg4)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathErrorStatus , arg2, arg3, arg4);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathGetRouteComplete +// [data][%p] Query route result: %!ADDR! +// QuicTraceEvent( + DatapathGetRouteComplete, + "[data][%p] Query route result: %!ADDR!", + Socket, + CASTED_CLOG_BYTEARRAY(sizeof(LocalAddress), &LocalAddress)); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = CASTED_CLOG_BYTEARRAY(sizeof(LocalAddress), &LocalAddress) = arg3 +----------------------------------------------------------*/ +#ifndef _clog_5_ARGS_TRACE_DatapathGetRouteComplete +#define _clog_5_ARGS_TRACE_DatapathGetRouteComplete(uniqueId, encoded_arg_string, arg2, arg3, arg3_len)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathGetRouteComplete , arg2, arg3_len, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathError +// [data][%p] ERROR, %s. +// QuicTraceEvent( + DatapathError, + "[data][%p] ERROR, %s.", + Socket, + "no matching interface/queue"); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = "no matching interface/queue" = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_DatapathError +#define _clog_4_ARGS_TRACE_DatapathError(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathError , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "CXPLAT_DATAPATH", + sizeof(CXPLAT_ROUTE_RESOLUTION_OPERATION)); +// arg2 = arg2 = "CXPLAT_DATAPATH" = arg2 +// arg3 = arg3 = sizeof(CXPLAT_ROUTE_RESOLUTION_OPERATION) = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_AllocFailure +#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, AllocFailure , arg2, arg3);\ + +#endif + + + + +#ifdef __cplusplus +} +#endif +#ifdef CLOG_INLINE_IMPLEMENTATION +#include "quic.clog_datapath_raw_socket_winuser.c.clog.h.c" +#endif diff --git a/src/generated/linux/datapath_raw_socket_winuser.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_socket_winuser.c.clog.h.lttng.h new file mode 100644 index 0000000000..2234bce3fb --- /dev/null +++ b/src/generated/linux/datapath_raw_socket_winuser.c.clog.h.lttng.h @@ -0,0 +1,179 @@ + + + +/*---------------------------------------------------------- +// Decoder Ring for RouteResolutionStart +// [conn][%p] Starting to look up neighbor on Path[%hhu] with status %u +// QuicTraceLogConnInfo( + RouteResolutionStart, + Context, + "Starting to look up neighbor on Path[%hhu] with status %u", + PathId, + Status); +// arg1 = arg1 = Context = arg1 +// arg3 = arg3 = PathId = arg3 +// arg4 = arg4 = Status = arg4 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, RouteResolutionStart, + TP_ARGS( + const void *, arg1, + unsigned char, arg3, + unsigned int, arg4), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg1, (uint64_t)arg1) + ctf_integer(unsigned char, arg3, arg3) + ctf_integer(unsigned int, arg4, arg4) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryErrorStatus +// [ lib] ERROR, %u, %s. +// QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + WsaError, + "WSAStartup"); +// arg2 = arg2 = WsaError = arg2 +// arg3 = arg3 = "WSAStartup" = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, LibraryErrorStatus, + TP_ARGS( + unsigned int, arg2, + const char *, arg3), + TP_FIELDS( + ctf_integer(unsigned int, arg2, arg2) + ctf_string(arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathGetRouteStart +// [data][%p] Querying route, local=%!ADDR!, remote=%!ADDR! +// QuicTraceEvent( + DatapathGetRouteStart, + "[data][%p] Querying route, local=%!ADDR!, remote=%!ADDR!", + Socket, + CASTED_CLOG_BYTEARRAY(sizeof(Route->LocalAddress), &Route->LocalAddress), + CASTED_CLOG_BYTEARRAY(sizeof(Route->RemoteAddress), &Route->RemoteAddress)); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = CASTED_CLOG_BYTEARRAY(sizeof(Route->LocalAddress), &Route->LocalAddress) = arg3 +// arg4 = arg4 = CASTED_CLOG_BYTEARRAY(sizeof(Route->RemoteAddress), &Route->RemoteAddress) = arg4 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathGetRouteStart, + TP_ARGS( + const void *, arg2, + unsigned int, arg3_len, + const void *, arg3, + unsigned int, arg4_len, + const void *, arg4), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3_len, arg3_len) + ctf_sequence(char, arg3, arg3, unsigned int, arg3_len) + ctf_integer(unsigned int, arg4_len, arg4_len) + ctf_sequence(char, arg4, arg4, unsigned int, arg4_len) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathErrorStatus +// [data][%p] ERROR, %u, %s. +// QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Status, + "GetBestRoute2"); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = Status = arg3 +// arg4 = arg4 = "GetBestRoute2" = arg4 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathErrorStatus, + TP_ARGS( + const void *, arg2, + unsigned int, arg3, + const char *, arg4), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ctf_string(arg4, arg4) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathGetRouteComplete +// [data][%p] Query route result: %!ADDR! +// QuicTraceEvent( + DatapathGetRouteComplete, + "[data][%p] Query route result: %!ADDR!", + Socket, + CASTED_CLOG_BYTEARRAY(sizeof(LocalAddress), &LocalAddress)); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = CASTED_CLOG_BYTEARRAY(sizeof(LocalAddress), &LocalAddress) = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathGetRouteComplete, + TP_ARGS( + const void *, arg2, + unsigned int, arg3_len, + const void *, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3_len, arg3_len) + ctf_sequence(char, arg3, arg3, unsigned int, arg3_len) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for DatapathError +// [data][%p] ERROR, %s. +// QuicTraceEvent( + DatapathError, + "[data][%p] ERROR, %s.", + Socket, + "no matching interface/queue"); +// arg2 = arg2 = Socket = arg2 +// arg3 = arg3 = "no matching interface/queue" = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, DatapathError, + TP_ARGS( + const void *, arg2, + const char *, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_string(arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "CXPLAT_DATAPATH", + sizeof(CXPLAT_ROUTE_RESOLUTION_OPERATION)); +// arg2 = arg2 = "CXPLAT_DATAPATH" = arg2 +// arg3 = arg3 = sizeof(CXPLAT_ROUTE_RESOLUTION_OPERATION) = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WINUSER_C, AllocFailure, + TP_ARGS( + const char *, arg2, + unsigned long long, arg3), + TP_FIELDS( + ctf_string(arg2, arg2) + ctf_integer(uint64_t, arg3, arg3) + ) +) diff --git a/src/generated/linux/datapath_raw_xdp_win.c.clog.h b/src/generated/linux/datapath_raw_xdp_win.c.clog.h deleted file mode 100644 index cb638745e2..0000000000 --- a/src/generated/linux/datapath_raw_xdp_win.c.clog.h +++ /dev/null @@ -1,379 +0,0 @@ -#ifndef CLOG_DO_NOT_INCLUDE_HEADER -#include -#endif -#undef TRACEPOINT_PROVIDER -#define TRACEPOINT_PROVIDER CLOG_DATAPATH_RAW_XDP_WIN_C -#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE -#define TRACEPOINT_PROBE_DYNAMIC_LINKAGE -#undef TRACEPOINT_INCLUDE -#define TRACEPOINT_INCLUDE "datapath_raw_xdp_win.c.clog.h.lttng.h" -#if !defined(DEF_CLOG_DATAPATH_RAW_XDP_WIN_C) || defined(TRACEPOINT_HEADER_MULTI_READ) -#define DEF_CLOG_DATAPATH_RAW_XDP_WIN_C -#include -#define __int64 __int64_t -#include "datapath_raw_xdp_win.c.clog.h.lttng.h" -#endif -#include -#ifndef _clog_MACRO_QuicTraceLogInfo -#define _clog_MACRO_QuicTraceLogInfo 1 -#define QuicTraceLogInfo(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) -#endif -#ifndef _clog_MACRO_QuicTraceLogVerbose -#define _clog_MACRO_QuicTraceLogVerbose 1 -#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) -#endif -#ifndef _clog_MACRO_QuicTraceEvent -#define _clog_MACRO_QuicTraceEvent 1 -#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) -#endif -#ifdef __cplusplus -extern "C" { -#endif -/*---------------------------------------------------------- -// Decoder Ring for FoundVF -// [ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu -// QuicTraceLogInfo( - FoundVF, - "[ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu", - Xdp, - Interface->IfIndex, - Interface->ActualIfIndex); -// arg2 = arg2 = Xdp = arg2 -// arg3 = arg3 = Interface->IfIndex = arg3 -// arg4 = arg4 = Interface->ActualIfIndex = arg4 -----------------------------------------------------------*/ -#ifndef _clog_5_ARGS_TRACE_FoundVF -#define _clog_5_ARGS_TRACE_FoundVF(uniqueId, encoded_arg_string, arg2, arg3, arg4)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, FoundVF , arg2, arg3, arg4);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpInterfaceQueues -// [ixdp][%p] Initializing %u queues on interface -// QuicTraceLogVerbose( - XdpInterfaceQueues, - "[ixdp][%p] Initializing %u queues on interface", - Interface, - Interface->QueueCount); -// arg2 = arg2 = Interface = arg2 -// arg3 = arg3 = Interface->QueueCount = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_XdpInterfaceQueues -#define _clog_4_ARGS_TRACE_XdpInterfaceQueues(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInterfaceQueues , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpInterfaceInitialize -// [ixdp][%p] Initializing interface %u -// QuicTraceLogVerbose( - XdpInterfaceInitialize, - "[ixdp][%p] Initializing interface %u", - Interface, - Interface->ActualIfIndex); -// arg2 = arg2 = Interface = arg2 -// arg3 = arg3 = Interface->ActualIfIndex = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_XdpInterfaceInitialize -#define _clog_4_ARGS_TRACE_XdpInterfaceInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInterfaceInitialize , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueStart -// [ xdp][%p] XDP queue start on partition %p -// QuicTraceLogVerbose( - XdpQueueStart, - "[ xdp][%p] XDP queue start on partition %p", - Queue, - Partition); -// arg2 = arg2 = Queue = arg2 -// arg3 = arg3 = Partition = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_XdpQueueStart -#define _clog_4_ARGS_TRACE_XdpQueueStart(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueStart , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpWorkerStart -// [ xdp][%p] XDP partition start, %u queues -// QuicTraceLogVerbose( - XdpWorkerStart, - "[ xdp][%p] XDP partition start, %u queues", - Partition, - QueueCount); -// arg2 = arg2 = Partition = arg2 -// arg3 = arg3 = QueueCount = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_XdpWorkerStart -#define _clog_4_ARGS_TRACE_XdpWorkerStart(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpWorkerStart , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpInitialize -// [ xdp][%p] XDP initialized, %u procs -// QuicTraceLogVerbose( - XdpInitialize, - "[ xdp][%p] XDP initialized, %u procs", - Xdp, - Xdp->PartitionCount); -// arg2 = arg2 = Xdp = arg2 -// arg3 = arg3 = Xdp->PartitionCount = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_XdpInitialize -#define _clog_4_ARGS_TRACE_XdpInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInitialize , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpRelease -// [ xdp][%p] XDP release -// QuicTraceLogVerbose( - XdpRelease, - "[ xdp][%p] XDP release", - Xdp); -// arg2 = arg2 = Xdp = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpRelease -#define _clog_3_ARGS_TRACE_XdpRelease(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpRelease , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpUninitializeComplete -// [ xdp][%p] XDP uninitialize complete -// QuicTraceLogVerbose( - XdpUninitializeComplete, - "[ xdp][%p] XDP uninitialize complete", - Xdp); -// arg2 = arg2 = Xdp = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpUninitializeComplete -#define _clog_3_ARGS_TRACE_XdpUninitializeComplete(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpUninitializeComplete , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpUninitialize -// [ xdp][%p] XDP uninitialize -// QuicTraceLogVerbose( - XdpUninitialize, - "[ xdp][%p] XDP uninitialize", - Xdp); -// arg2 = arg2 = Xdp = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpUninitialize -#define _clog_3_ARGS_TRACE_XdpUninitialize(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpUninitialize , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpPartitionShutdown -// [ xdp][%p] XDP partition shutdown -// QuicTraceLogVerbose( - XdpPartitionShutdown, - "[ xdp][%p] XDP partition shutdown", - Partition); -// arg2 = arg2 = Partition = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpPartitionShutdown -#define _clog_3_ARGS_TRACE_XdpPartitionShutdown(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdown , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoRx -// [ xdp][%p] XDP async IO start (RX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoRx, - "[ xdp][%p] XDP async IO start (RX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpQueueAsyncIoRx -#define _clog_3_ARGS_TRACE_XdpQueueAsyncIoRx(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoRx , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoTx -// [ xdp][%p] XDP async IO start (TX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoTx, - "[ xdp][%p] XDP async IO start (TX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpQueueAsyncIoTx -#define _clog_3_ARGS_TRACE_XdpQueueAsyncIoTx(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoTx , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoRxComplete -// [ xdp][%p] XDP async IO complete (RX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoRxComplete, - "[ xdp][%p] XDP async IO complete (RX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpQueueAsyncIoRxComplete -#define _clog_3_ARGS_TRACE_XdpQueueAsyncIoRxComplete(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoRxComplete , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoTxComplete -// [ xdp][%p] XDP async IO complete (TX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoTxComplete, - "[ xdp][%p] XDP async IO complete (TX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpQueueAsyncIoTxComplete -#define _clog_3_ARGS_TRACE_XdpQueueAsyncIoTxComplete(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoTxComplete , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpPartitionShutdownComplete -// [ xdp][%p] XDP partition shutdown complete -// QuicTraceLogVerbose( - XdpPartitionShutdownComplete, - "[ xdp][%p] XDP partition shutdown complete", - Partition); -// arg2 = arg2 = Partition = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_XdpPartitionShutdownComplete -#define _clog_3_ARGS_TRACE_XdpPartitionShutdownComplete(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdownComplete , arg2);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for LibraryErrorStatus -// [ lib] ERROR, %u, %s. -// QuicTraceEvent( - LibraryErrorStatus, - "[ lib] ERROR, %u, %s.", - Status, - "XskBind (GetRssQueueProcessors)"); -// arg2 = arg2 = Status = arg2 -// arg3 = arg3 = "XskBind (GetRssQueueProcessors)" = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_LibraryErrorStatus -#define _clog_4_ARGS_TRACE_LibraryErrorStatus(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, LibraryErrorStatus , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for AllocFailure -// Allocation of '%s' failed. (%llu bytes) -// QuicTraceEvent( - AllocFailure, - "Allocation of '%s' failed. (%llu bytes)", - "XDP Queues", - Interface->QueueCount * sizeof(*Interface->Queues)); -// arg2 = arg2 = "XDP Queues" = arg2 -// arg3 = arg3 = Interface->QueueCount * sizeof(*Interface->Queues) = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_AllocFailure -#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, AllocFailure , arg2, arg3);\ - -#endif - - - - -/*---------------------------------------------------------- -// Decoder Ring for LibraryError -// [ lib] ERROR, %s. -// QuicTraceEvent( - LibraryError, - "[ lib] ERROR, %s.", - "No more room for rules"); -// arg2 = arg2 = "No more room for rules" = arg2 -----------------------------------------------------------*/ -#ifndef _clog_3_ARGS_TRACE_LibraryError -#define _clog_3_ARGS_TRACE_LibraryError(uniqueId, encoded_arg_string, arg2)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, LibraryError , arg2);\ - -#endif - - - - -#ifdef __cplusplus -} -#endif -#ifdef CLOG_INLINE_IMPLEMENTATION -#include "quic.clog_datapath_raw_xdp_win.c.clog.h.c" -#endif diff --git a/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h deleted file mode 100644 index 13fb31dd19..0000000000 --- a/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h +++ /dev/null @@ -1,378 +0,0 @@ - - - -/*---------------------------------------------------------- -// Decoder Ring for FoundVF -// [ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu -// QuicTraceLogInfo( - FoundVF, - "[ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu", - Xdp, - Interface->IfIndex, - Interface->ActualIfIndex); -// arg2 = arg2 = Xdp = arg2 -// arg3 = arg3 = Interface->IfIndex = arg3 -// arg4 = arg4 = Interface->ActualIfIndex = arg4 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, FoundVF, - TP_ARGS( - const void *, arg2, - unsigned int, arg3, - unsigned int, arg4), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer(unsigned int, arg3, arg3) - ctf_integer(unsigned int, arg4, arg4) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpInterfaceQueues -// [ixdp][%p] Initializing %u queues on interface -// QuicTraceLogVerbose( - XdpInterfaceQueues, - "[ixdp][%p] Initializing %u queues on interface", - Interface, - Interface->QueueCount); -// arg2 = arg2 = Interface = arg2 -// arg3 = arg3 = Interface->QueueCount = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInterfaceQueues, - TP_ARGS( - const void *, arg2, - unsigned int, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer(unsigned int, arg3, arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpInterfaceInitialize -// [ixdp][%p] Initializing interface %u -// QuicTraceLogVerbose( - XdpInterfaceInitialize, - "[ixdp][%p] Initializing interface %u", - Interface, - Interface->ActualIfIndex); -// arg2 = arg2 = Interface = arg2 -// arg3 = arg3 = Interface->ActualIfIndex = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInterfaceInitialize, - TP_ARGS( - const void *, arg2, - unsigned int, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer(unsigned int, arg3, arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueStart -// [ xdp][%p] XDP queue start on partition %p -// QuicTraceLogVerbose( - XdpQueueStart, - "[ xdp][%p] XDP queue start on partition %p", - Queue, - Partition); -// arg2 = arg2 = Queue = arg2 -// arg3 = arg3 = Partition = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueStart, - TP_ARGS( - const void *, arg2, - const void *, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer_hex(uint64_t, arg3, (uint64_t)arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpWorkerStart -// [ xdp][%p] XDP partition start, %u queues -// QuicTraceLogVerbose( - XdpWorkerStart, - "[ xdp][%p] XDP partition start, %u queues", - Partition, - QueueCount); -// arg2 = arg2 = Partition = arg2 -// arg3 = arg3 = QueueCount = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpWorkerStart, - TP_ARGS( - const void *, arg2, - unsigned int, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer(unsigned int, arg3, arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpInitialize -// [ xdp][%p] XDP initialized, %u procs -// QuicTraceLogVerbose( - XdpInitialize, - "[ xdp][%p] XDP initialized, %u procs", - Xdp, - Xdp->PartitionCount); -// arg2 = arg2 = Xdp = arg2 -// arg3 = arg3 = Xdp->PartitionCount = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInitialize, - TP_ARGS( - const void *, arg2, - unsigned int, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer(unsigned int, arg3, arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpRelease -// [ xdp][%p] XDP release -// QuicTraceLogVerbose( - XdpRelease, - "[ xdp][%p] XDP release", - Xdp); -// arg2 = arg2 = Xdp = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpRelease, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpUninitializeComplete -// [ xdp][%p] XDP uninitialize complete -// QuicTraceLogVerbose( - XdpUninitializeComplete, - "[ xdp][%p] XDP uninitialize complete", - Xdp); -// arg2 = arg2 = Xdp = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpUninitializeComplete, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpUninitialize -// [ xdp][%p] XDP uninitialize -// QuicTraceLogVerbose( - XdpUninitialize, - "[ xdp][%p] XDP uninitialize", - Xdp); -// arg2 = arg2 = Xdp = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpUninitialize, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpPartitionShutdown -// [ xdp][%p] XDP partition shutdown -// QuicTraceLogVerbose( - XdpPartitionShutdown, - "[ xdp][%p] XDP partition shutdown", - Partition); -// arg2 = arg2 = Partition = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdown, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoRx -// [ xdp][%p] XDP async IO start (RX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoRx, - "[ xdp][%p] XDP async IO start (RX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoRx, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoTx -// [ xdp][%p] XDP async IO start (TX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoTx, - "[ xdp][%p] XDP async IO start (TX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoTx, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoRxComplete -// [ xdp][%p] XDP async IO complete (RX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoRxComplete, - "[ xdp][%p] XDP async IO complete (RX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoRxComplete, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpQueueAsyncIoTxComplete -// [ xdp][%p] XDP async IO complete (TX) -// QuicTraceLogVerbose( - XdpQueueAsyncIoTxComplete, - "[ xdp][%p] XDP async IO complete (TX)", - Queue); -// arg2 = arg2 = Queue = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpQueueAsyncIoTxComplete, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for XdpPartitionShutdownComplete -// [ xdp][%p] XDP partition shutdown complete -// QuicTraceLogVerbose( - XdpPartitionShutdownComplete, - "[ xdp][%p] XDP partition shutdown complete", - Partition); -// arg2 = arg2 = Partition = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdownComplete, - TP_ARGS( - const void *, arg2), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for LibraryErrorStatus -// [ lib] ERROR, %u, %s. -// QuicTraceEvent( - LibraryErrorStatus, - "[ lib] ERROR, %u, %s.", - Status, - "XskBind (GetRssQueueProcessors)"); -// arg2 = arg2 = Status = arg2 -// arg3 = arg3 = "XskBind (GetRssQueueProcessors)" = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, LibraryErrorStatus, - TP_ARGS( - unsigned int, arg2, - const char *, arg3), - TP_FIELDS( - ctf_integer(unsigned int, arg2, arg2) - ctf_string(arg3, arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for AllocFailure -// Allocation of '%s' failed. (%llu bytes) -// QuicTraceEvent( - AllocFailure, - "Allocation of '%s' failed. (%llu bytes)", - "XDP Queues", - Interface->QueueCount * sizeof(*Interface->Queues)); -// arg2 = arg2 = "XDP Queues" = arg2 -// arg3 = arg3 = Interface->QueueCount * sizeof(*Interface->Queues) = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, AllocFailure, - TP_ARGS( - const char *, arg2, - unsigned long long, arg3), - TP_FIELDS( - ctf_string(arg2, arg2) - ctf_integer(uint64_t, arg3, arg3) - ) -) - - - -/*---------------------------------------------------------- -// Decoder Ring for LibraryError -// [ lib] ERROR, %s. -// QuicTraceEvent( - LibraryError, - "[ lib] ERROR, %s.", - "No more room for rules"); -// arg2 = arg2 = "No more room for rules" = arg2 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, LibraryError, - TP_ARGS( - const char *, arg2), - TP_FIELDS( - ctf_string(arg2, arg2) - ) -) diff --git a/src/generated/linux/datapath_raw_xdp_winkernel.c.clog.h b/src/generated/linux/datapath_raw_xdp_winkernel.c.clog.h new file mode 100644 index 0000000000..d17bd3a4bf --- /dev/null +++ b/src/generated/linux/datapath_raw_xdp_winkernel.c.clog.h @@ -0,0 +1,171 @@ +#ifndef CLOG_DO_NOT_INCLUDE_HEADER +#include +#endif +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER CLOG_DATAPATH_RAW_XDP_WINKERNEL_C +#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#define TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "datapath_raw_xdp_winkernel.c.clog.h.lttng.h" +#if !defined(DEF_CLOG_DATAPATH_RAW_XDP_WINKERNEL_C) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define DEF_CLOG_DATAPATH_RAW_XDP_WINKERNEL_C +#include +#define __int64 __int64_t +#include "datapath_raw_xdp_winkernel.c.clog.h.lttng.h" +#endif +#include +#ifndef _clog_MACRO_QuicTraceLogVerbose +#define _clog_MACRO_QuicTraceLogVerbose 1 +#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifndef _clog_MACRO_QuicTraceEvent +#define _clog_MACRO_QuicTraceEvent 1 +#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifdef __cplusplus +extern "C" { +#endif +/*---------------------------------------------------------- +// Decoder Ring for XdpInterfaceInitialize +// [ixdp][%p] Initializing interface %u +// QuicTraceLogVerbose( + XdpInterfaceInitialize, + "[ixdp][%p] Initializing interface %u", + Interface, + Interface->ActualIfIndex); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->ActualIfIndex = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpInterfaceInitialize +#define _clog_4_ARGS_TRACE_XdpInterfaceInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpInterfaceInitialize , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpQueueStart +// [ xdp][%p] XDP queue start on partition %p +// QuicTraceLogVerbose( + XdpQueueStart, + "[ xdp][%p] XDP queue start on partition %p", + Queue, + Partition); +// arg2 = arg2 = Queue = arg2 +// arg3 = arg3 = Partition = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpQueueStart +#define _clog_4_ARGS_TRACE_XdpQueueStart(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpQueueStart , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpWorkerStart +// [ xdp][%p] XDP partition start, %u queues +// QuicTraceLogVerbose( + XdpWorkerStart, + "[ xdp][%p] XDP partition start, %u queues", + Partition, + QueueCount); +// arg2 = arg2 = Partition = arg2 +// arg3 = arg3 = QueueCount = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpWorkerStart +#define _clog_4_ARGS_TRACE_XdpWorkerStart(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpWorkerStart , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInitialize +// [ xdp][%p] XDP initialized, %u procs +// QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Xdp->PartitionCount = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpInitialize +#define _clog_4_ARGS_TRACE_XdpInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpInitialize , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + sizeof(*Interface)); +// arg2 = arg2 = "XDP interface" = arg2 +// arg3 = arg3 = sizeof(*Interface) = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_AllocFailure +#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, AllocFailure , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryErrorStatus +// [ lib] ERROR, %u, %s. +// QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatDpRawInterfaceInitialize"); +// arg2 = arg2 = Status = arg2 +// arg3 = arg3 = "CxPlatDpRawInterfaceInitialize" = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_LibraryErrorStatus +#define _clog_4_ARGS_TRACE_LibraryErrorStatus(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, LibraryErrorStatus , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryError +// [ lib] ERROR, %s. +// QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); +// arg2 = arg2 = "no XDP capable interface" = arg2 +----------------------------------------------------------*/ +#ifndef _clog_3_ARGS_TRACE_LibraryError +#define _clog_3_ARGS_TRACE_LibraryError(uniqueId, encoded_arg_string, arg2)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, LibraryError , arg2);\ + +#endif + + + + +#ifdef __cplusplus +} +#endif +#ifdef CLOG_INLINE_IMPLEMENTATION +#include "quic.clog_datapath_raw_xdp_winkernel.c.clog.h.c" +#endif diff --git a/src/generated/linux/datapath_raw_xdp_winkernel.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_xdp_winkernel.c.clog.h.lttng.h new file mode 100644 index 0000000000..ec3d830c7f --- /dev/null +++ b/src/generated/linux/datapath_raw_xdp_winkernel.c.clog.h.lttng.h @@ -0,0 +1,157 @@ + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInterfaceInitialize +// [ixdp][%p] Initializing interface %u +// QuicTraceLogVerbose( + XdpInterfaceInitialize, + "[ixdp][%p] Initializing interface %u", + Interface, + Interface->ActualIfIndex); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->ActualIfIndex = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpInterfaceInitialize, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpQueueStart +// [ xdp][%p] XDP queue start on partition %p +// QuicTraceLogVerbose( + XdpQueueStart, + "[ xdp][%p] XDP queue start on partition %p", + Queue, + Partition); +// arg2 = arg2 = Queue = arg2 +// arg3 = arg3 = Partition = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpQueueStart, + TP_ARGS( + const void *, arg2, + const void *, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer_hex(uint64_t, arg3, (uint64_t)arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpWorkerStart +// [ xdp][%p] XDP partition start, %u queues +// QuicTraceLogVerbose( + XdpWorkerStart, + "[ xdp][%p] XDP partition start, %u queues", + Partition, + QueueCount); +// arg2 = arg2 = Partition = arg2 +// arg3 = arg3 = QueueCount = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpWorkerStart, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInitialize +// [ xdp][%p] XDP initialized, %u procs +// QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Xdp->PartitionCount = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, XdpInitialize, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + sizeof(*Interface)); +// arg2 = arg2 = "XDP interface" = arg2 +// arg3 = arg3 = sizeof(*Interface) = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, AllocFailure, + TP_ARGS( + const char *, arg2, + unsigned long long, arg3), + TP_FIELDS( + ctf_string(arg2, arg2) + ctf_integer(uint64_t, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryErrorStatus +// [ lib] ERROR, %u, %s. +// QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatDpRawInterfaceInitialize"); +// arg2 = arg2 = Status = arg2 +// arg3 = arg3 = "CxPlatDpRawInterfaceInitialize" = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, LibraryErrorStatus, + TP_ARGS( + unsigned int, arg2, + const char *, arg3), + TP_FIELDS( + ctf_integer(unsigned int, arg2, arg2) + ctf_string(arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryError +// [ lib] ERROR, %s. +// QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); +// arg2 = arg2 = "no XDP capable interface" = arg2 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINKERNEL_C, LibraryError, + TP_ARGS( + const char *, arg2), + TP_FIELDS( + ctf_string(arg2, arg2) + ) +) diff --git a/src/generated/linux/datapath_raw_xdp_winuser.c.clog.h b/src/generated/linux/datapath_raw_xdp_winuser.c.clog.h new file mode 100644 index 0000000000..1e03b85c2a --- /dev/null +++ b/src/generated/linux/datapath_raw_xdp_winuser.c.clog.h @@ -0,0 +1,197 @@ +#ifndef CLOG_DO_NOT_INCLUDE_HEADER +#include +#endif +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER CLOG_DATAPATH_RAW_XDP_WINUSER_C +#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#define TRACEPOINT_PROBE_DYNAMIC_LINKAGE +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "datapath_raw_xdp_winuser.c.clog.h.lttng.h" +#if !defined(DEF_CLOG_DATAPATH_RAW_XDP_WINUSER_C) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define DEF_CLOG_DATAPATH_RAW_XDP_WINUSER_C +#include +#define __int64 __int64_t +#include "datapath_raw_xdp_winuser.c.clog.h.lttng.h" +#endif +#include +#ifndef _clog_MACRO_QuicTraceLogInfo +#define _clog_MACRO_QuicTraceLogInfo 1 +#define QuicTraceLogInfo(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifndef _clog_MACRO_QuicTraceLogVerbose +#define _clog_MACRO_QuicTraceLogVerbose 1 +#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifndef _clog_MACRO_QuicTraceEvent +#define _clog_MACRO_QuicTraceEvent 1 +#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif +#ifdef __cplusplus +extern "C" { +#endif +/*---------------------------------------------------------- +// Decoder Ring for FoundVF +// [ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu +// QuicTraceLogInfo( + FoundVF, + "[ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu", + Xdp, + Interface->IfIndex, + Interface->ActualIfIndex); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Interface->IfIndex = arg3 +// arg4 = arg4 = Interface->ActualIfIndex = arg4 +----------------------------------------------------------*/ +#ifndef _clog_5_ARGS_TRACE_FoundVF +#define _clog_5_ARGS_TRACE_FoundVF(uniqueId, encoded_arg_string, arg2, arg3, arg4)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, FoundVF , arg2, arg3, arg4);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInterfaceInitialize +// [ixdp][%p] Initializing interface %u +// QuicTraceLogVerbose( + XdpInterfaceInitialize, + "[ixdp][%p] Initializing interface %u", + Interface, + Interface->ActualIfIndex); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->ActualIfIndex = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpInterfaceInitialize +#define _clog_4_ARGS_TRACE_XdpInterfaceInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpInterfaceInitialize , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpQueueStart +// [ xdp][%p] XDP queue start on partition %p +// QuicTraceLogVerbose( + XdpQueueStart, + "[ xdp][%p] XDP queue start on partition %p", + Queue, + Partition); +// arg2 = arg2 = Queue = arg2 +// arg3 = arg3 = Partition = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpQueueStart +#define _clog_4_ARGS_TRACE_XdpQueueStart(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpQueueStart , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpWorkerStart +// [ xdp][%p] XDP partition start, %u queues +// QuicTraceLogVerbose( + XdpWorkerStart, + "[ xdp][%p] XDP partition start, %u queues", + Partition, + QueueCount); +// arg2 = arg2 = Partition = arg2 +// arg3 = arg3 = QueueCount = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpWorkerStart +#define _clog_4_ARGS_TRACE_XdpWorkerStart(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpWorkerStart , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInitialize +// [ xdp][%p] XDP initialized, %u procs +// QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Xdp->PartitionCount = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpInitialize +#define _clog_4_ARGS_TRACE_XdpInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpInitialize , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + AdaptersBufferSize); +// arg2 = arg2 = "XDP interface" = arg2 +// arg3 = arg3 = AdaptersBufferSize = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_AllocFailure +#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, AllocFailure , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryErrorStatus +// [ lib] ERROR, %u, %s. +// QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatDpRawInterfaceInitialize"); +// arg2 = arg2 = Status = arg2 +// arg3 = arg3 = "CxPlatDpRawInterfaceInitialize" = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_LibraryErrorStatus +#define _clog_4_ARGS_TRACE_LibraryErrorStatus(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, LibraryErrorStatus , arg2, arg3);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryError +// [ lib] ERROR, %s. +// QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); +// arg2 = arg2 = "no XDP capable interface" = arg2 +----------------------------------------------------------*/ +#ifndef _clog_3_ARGS_TRACE_LibraryError +#define _clog_3_ARGS_TRACE_LibraryError(uniqueId, encoded_arg_string, arg2)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WINUSER_C, LibraryError , arg2);\ + +#endif + + + + +#ifdef __cplusplus +} +#endif +#ifdef CLOG_INLINE_IMPLEMENTATION +#include "quic.clog_datapath_raw_xdp_winuser.c.clog.h.c" +#endif diff --git a/src/generated/linux/datapath_raw_xdp_winuser.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_xdp_winuser.c.clog.h.lttng.h new file mode 100644 index 0000000000..6a00ac3dde --- /dev/null +++ b/src/generated/linux/datapath_raw_xdp_winuser.c.clog.h.lttng.h @@ -0,0 +1,184 @@ + + + +/*---------------------------------------------------------- +// Decoder Ring for FoundVF +// [ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu +// QuicTraceLogInfo( + FoundVF, + "[ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu", + Xdp, + Interface->IfIndex, + Interface->ActualIfIndex); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Interface->IfIndex = arg3 +// arg4 = arg4 = Interface->ActualIfIndex = arg4 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, FoundVF, + TP_ARGS( + const void *, arg2, + unsigned int, arg3, + unsigned int, arg4), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ctf_integer(unsigned int, arg4, arg4) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInterfaceInitialize +// [ixdp][%p] Initializing interface %u +// QuicTraceLogVerbose( + XdpInterfaceInitialize, + "[ixdp][%p] Initializing interface %u", + Interface, + Interface->ActualIfIndex); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->ActualIfIndex = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpInterfaceInitialize, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpQueueStart +// [ xdp][%p] XDP queue start on partition %p +// QuicTraceLogVerbose( + XdpQueueStart, + "[ xdp][%p] XDP queue start on partition %p", + Queue, + Partition); +// arg2 = arg2 = Queue = arg2 +// arg3 = arg3 = Partition = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpQueueStart, + TP_ARGS( + const void *, arg2, + const void *, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer_hex(uint64_t, arg3, (uint64_t)arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpWorkerStart +// [ xdp][%p] XDP partition start, %u queues +// QuicTraceLogVerbose( + XdpWorkerStart, + "[ xdp][%p] XDP partition start, %u queues", + Partition, + QueueCount); +// arg2 = arg2 = Partition = arg2 +// arg3 = arg3 = QueueCount = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpWorkerStart, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpInitialize +// [ xdp][%p] XDP initialized, %u procs +// QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Xdp->PartitionCount = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, XdpInitialize, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + AdaptersBufferSize); +// arg2 = arg2 = "XDP interface" = arg2 +// arg3 = arg3 = AdaptersBufferSize = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, AllocFailure, + TP_ARGS( + const char *, arg2, + unsigned long long, arg3), + TP_FIELDS( + ctf_string(arg2, arg2) + ctf_integer(uint64_t, arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryErrorStatus +// [ lib] ERROR, %u, %s. +// QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatDpRawInterfaceInitialize"); +// arg2 = arg2 = Status = arg2 +// arg3 = arg3 = "CxPlatDpRawInterfaceInitialize" = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, LibraryErrorStatus, + TP_ARGS( + unsigned int, arg2, + const char *, arg3), + TP_FIELDS( + ctf_integer(unsigned int, arg2, arg2) + ctf_string(arg3, arg3) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for LibraryError +// [ lib] ERROR, %s. +// QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); +// arg2 = arg2 = "no XDP capable interface" = arg2 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WINUSER_C, LibraryError, + TP_ARGS( + const char *, arg2), + TP_FIELDS( + ctf_string(arg2, arg2) + ) +) diff --git a/src/generated/linux/quic.clog_datapath_raw_socket_common.c.clog.h.c b/src/generated/linux/quic.clog_datapath_raw_socket_common.c.clog.h.c new file mode 100644 index 0000000000..e4a7ec01a8 --- /dev/null +++ b/src/generated/linux/quic.clog_datapath_raw_socket_common.c.clog.h.c @@ -0,0 +1,7 @@ +#include +#ifdef BUILDING_TRACEPOINT_PROVIDER +#define TRACEPOINT_CREATE_PROBES +#else +#define TRACEPOINT_DEFINE +#endif +#include "datapath_raw_socket_common.c.clog.h" diff --git a/src/generated/linux/quic.clog_datapath_raw_socket_winuser.c.clog.h.c b/src/generated/linux/quic.clog_datapath_raw_socket_winuser.c.clog.h.c new file mode 100644 index 0000000000..ca5228bb3c --- /dev/null +++ b/src/generated/linux/quic.clog_datapath_raw_socket_winuser.c.clog.h.c @@ -0,0 +1,7 @@ +#include +#ifdef BUILDING_TRACEPOINT_PROVIDER +#define TRACEPOINT_CREATE_PROBES +#else +#define TRACEPOINT_DEFINE +#endif +#include "datapath_raw_socket_winuser.c.clog.h" diff --git a/src/generated/linux/quic.clog_datapath_raw_xdp_winkernel.c.clog.h.c b/src/generated/linux/quic.clog_datapath_raw_xdp_winkernel.c.clog.h.c new file mode 100644 index 0000000000..64223136a9 --- /dev/null +++ b/src/generated/linux/quic.clog_datapath_raw_xdp_winkernel.c.clog.h.c @@ -0,0 +1,7 @@ +#include +#ifdef BUILDING_TRACEPOINT_PROVIDER +#define TRACEPOINT_CREATE_PROBES +#else +#define TRACEPOINT_DEFINE +#endif +#include "datapath_raw_xdp_winkernel.c.clog.h" diff --git a/src/generated/linux/quic.clog_datapath_raw_xdp_win.c.clog.h.c b/src/generated/linux/quic.clog_datapath_raw_xdp_winuser.c.clog.h.c similarity index 73% rename from src/generated/linux/quic.clog_datapath_raw_xdp_win.c.clog.h.c rename to src/generated/linux/quic.clog_datapath_raw_xdp_winuser.c.clog.h.c index e5fc9f1679..10c4aef4ab 100644 --- a/src/generated/linux/quic.clog_datapath_raw_xdp_win.c.clog.h.c +++ b/src/generated/linux/quic.clog_datapath_raw_xdp_winuser.c.clog.h.c @@ -4,4 +4,4 @@ #else #define TRACEPOINT_DEFINE #endif -#include "datapath_raw_xdp_win.c.clog.h" +#include "datapath_raw_xdp_winuser.c.clog.h" diff --git a/src/inc/quic_platform.h b/src/inc/quic_platform.h index 4e56a87fd5..90938dbc20 100644 --- a/src/inc/quic_platform.h +++ b/src/inc/quic_platform.h @@ -436,20 +436,17 @@ typedef struct CXPLAT_WORKER_POOL { } CXPLAT_WORKER_POOL; -#ifdef _KERNEL_MODE // Not supported on kernel mode -#define CxPlatWorkerPoolInit(WorkerPool) UNREFERENCED_PARAMETER(WorkerPool) -#define CxPlatWorkerPoolUninit(WorkerPool) UNREFERENCED_PARAMETER(WorkerPool) -#else +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatWorkerPoolInit( _In_ CXPLAT_WORKER_POOL* WorkerPool ); +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatWorkerPoolUninit( _In_ CXPLAT_WORKER_POOL* WorkerPool ); -#endif // // General purpose execution context abstraction layer. Used for driving worker @@ -524,10 +521,7 @@ typedef struct CXPLAT_EXECUTION_CONTEXT { } CXPLAT_EXECUTION_CONTEXT; -#ifdef _KERNEL_MODE // Not supported on kernel mode -#define CxPlatAddExecutionContext(WorkerPool, Context, IdealProcessor) CXPLAT_FRE_ASSERT(FALSE) -#define CxPlatWakeExecutionContext(Context) CXPLAT_FRE_ASSERT(FALSE) -#else +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatAddExecutionContext( _In_ CXPLAT_WORKER_POOL* WorkerPool, @@ -535,11 +529,11 @@ CxPlatAddExecutionContext( _In_ uint16_t Index // Into the execution config processor array ); +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatWakeExecutionContext( _In_ CXPLAT_EXECUTION_CONTEXT* Context ); -#endif // // Test Interface for loading a self-signed certificate. diff --git a/src/inc/quic_platform_posix.h b/src/inc/quic_platform_posix.h index f963770741..213f46162e 100644 --- a/src/inc/quic_platform_posix.h +++ b/src/inc/quic_platform_posix.h @@ -1003,6 +1003,8 @@ CxPlatEventQEnqueue( return TRUE; } +#define CXPLAT_EVENTQ_DEQUEUE_MAX 16 + inline uint32_t CxPlatEventQDequeue( @@ -1115,6 +1117,8 @@ CxPlatEventQEnqueue( return eventfd_write(sqe->fd, 1) == 0; } +#define CXPLAT_EVENTQ_DEQUEUE_MAX 16 + inline uint32_t CxPlatEventQDequeue( @@ -1241,6 +1245,8 @@ CxPlatEventQEnqueueEx( return kevent(*queue, &event, 1, NULL, 0, NULL) == 0; } +#define CXPLAT_EVENTQ_DEQUEUE_MAX 16 + inline uint32_t CxPlatEventQDequeue( diff --git a/src/inc/quic_platform_winkernel.h b/src/inc/quic_platform_winkernel.h index bb45554f80..f96ce2f67e 100644 --- a/src/inc/quic_platform_winkernel.h +++ b/src/inc/quic_platform_winkernel.h @@ -42,6 +42,7 @@ #include #include #include +#include #include "msquic_winkernel.h" #pragma warning(pop) @@ -130,6 +131,27 @@ InterlockedFetchAndSetBoolean( return (BOOLEAN)InterlockedOr8((char*)Target, 1); } +_IRQL_requires_max_(PASSIVE_LEVEL) +inline +void +CxPlatCloseHandle( + _Pre_notnull_ HANDLE Handle + ) +{ + ZwClose(Handle); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +inline +void +CxPlatCancelIo( + _Pre_notnull_ HANDLE Handle + ) +{ + // TODO: Implement + UNREFERENCED_PARAMETER(Handle); +} + // // Static Analysis Interfaces // @@ -469,58 +491,242 @@ _CxPlatEventWaitWithTimeout( #define CxPlatEventWaitWithTimeout(Event, TimeoutMs) \ (STATUS_SUCCESS == _CxPlatEventWaitWithTimeout(&Event, TimeoutMs)) +// +// TODO: Publish and document the following APIs +// + +// ACCESS_MASK +#define IO_COMPLETION_QUERY_STATE 0x0001 +#define IO_COMPLETION_MODIFY_STATE 0x0002 // winnt +#define IO_COMPLETION_ALL_ACCESS (STANDARD_RIGHTS_REQUIRED|SYNCHRONIZE|0x3) // winnt + +typedef struct _FILE_IO_COMPLETION_INFORMATION { + PVOID KeyContext; + PVOID ApcContext; + IO_STATUS_BLOCK IoStatusBlock; +} FILE_IO_COMPLETION_INFORMATION, *PFILE_IO_COMPLETION_INFORMATION; + +NTSYSAPI +NTSTATUS +NTAPI +ZwCreateIoCompletion( + OUT PHANDLE IoCompletionHandle, + IN ACCESS_MASK DesiredAccess, + IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, + IN OPTIONAL ULONG Count + ); + +typedef struct _IO_MINI_COMPLETION_PACKET_USER IO_MINI_COMPLETION_PACKET_USER, *PIO_MINI_COMPLETION_PACKET_USER; + +typedef +VOID +IO_MINI_PACKET_CALLBACK_ROUTINE( + _In_ PIO_MINI_COMPLETION_PACKET_USER MiniPacket, + _In_opt_ PVOID Context + ); + +typedef IO_MINI_PACKET_CALLBACK_ROUTINE *PIO_MINI_PACKET_CALLBACK_ROUTINE; + +NTSYSAPI +NTSTATUS +IoSetIoCompletionEx( + _In_ PVOID IoCompletion, + _In_opt_ PVOID KeyContext, + _In_opt_ PVOID ApcContext, + _In_ NTSTATUS IoStatus, + _In_ ULONG_PTR IoStatusInformation, + _In_ BOOLEAN Quota, + _In_opt_ PIO_MINI_COMPLETION_PACKET_USER MiniPacket + ); + +NTSYSAPI +NTSTATUS +IoRemoveIoCompletion( + _In_ PVOID IoCompletionPtr, + _Out_writes_(Count) PFILE_IO_COMPLETION_INFORMATION IoCompletionInformation, + _Out_writes_(Count) PLIST_ENTRY *EntryArray, + _In_ ULONG Count, + _Out_ PULONG NumEntriesRemoved, + _In_ KPROCESSOR_MODE PreviousMode, + _In_opt_ PLARGE_INTEGER CapturedTimeout, + _In_ BOOLEAN Alertable + ); + +NTSYSAPI +PIO_MINI_COMPLETION_PACKET_USER +IoAllocateMiniCompletionPacket( + __in PIO_MINI_PACKET_CALLBACK_ROUTINE CallbackRoutine, + __in_opt PVOID Context + ); + +NTSYSAPI +VOID +IoFreeMiniCompletionPacket( + __inout PIO_MINI_COMPLETION_PACKET_USER MiniPacket + ); + // // Event Queue Interfaces // -typedef KEVENT CXPLAT_EVENTQ; // Event queue -typedef void* CXPLAT_CQE; +typedef struct CXPLAT_EVENTQ { + PKQUEUE IoCompletion; +} CXPLAT_EVENTQ; + +typedef FILE_IO_COMPLETION_INFORMATION CXPLAT_CQE; +typedef +_IRQL_requires_max_(PASSIVE_LEVEL) +void +(CXPLAT_EVENT_COMPLETION)( + _In_ CXPLAT_CQE* Cqe + ); +typedef CXPLAT_EVENT_COMPLETION *CXPLAT_EVENT_COMPLETION_HANDLER; + +typedef struct CXPLAT_SQE { + PIO_MINI_COMPLETION_PACKET_USER MiniPacket; + CXPLAT_EVENT_COMPLETION_HANDLER Completion; +} CXPLAT_SQE; + +inline +VOID +IoMiniPacketCallbackRoutineNoOp( + _In_ struct _IO_MINI_COMPLETION_PACKET_USER * MiniPacket, + _In_opt_ PVOID Context + ) +{ + UNREFERENCED_PARAMETER(MiniPacket); + UNREFERENCED_PARAMETER(Context); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +_Success_(return != FALSE) inline BOOLEAN CxPlatEventQInitialize( _Out_ CXPLAT_EVENTQ* queue ) { - KeInitializeEvent(queue, SynchronizationEvent, FALSE); - return TRUE; + HANDLE Handle; + OBJECT_ATTRIBUTES KernelObjectAttributes; + InitializeObjectAttributes( + &KernelObjectAttributes, + NULL, + OBJ_KERNEL_HANDLE, + NULL, + NULL); + + NTSTATUS Status = + ZwCreateIoCompletion( + &Handle, + IO_COMPLETION_ALL_ACCESS, + &KernelObjectAttributes, + 0); + if (!NT_SUCCESS(Status)) { + return FALSE; + } + + Status = + ObReferenceObjectByHandle( + Handle, + IO_COMPLETION_ALL_ACCESS, + NULL, + KernelMode, + (PVOID*)&queue->IoCompletion, + NULL); + + NtClose(Handle); + return NT_SUCCESS(Status); } +_IRQL_requires_max_(PASSIVE_LEVEL) inline void CxPlatEventQCleanup( _In_ CXPLAT_EVENTQ* queue ) { - UNREFERENCED_PARAMETER(queue); + ObDereferenceObject(queue->IoCompletion); } +_IRQL_requires_max_(DISPATCH_LEVEL) inline BOOLEAN -_CxPlatEventQEnqueue( - _In_ CXPLAT_EVENTQ* queue +CxPlatEventQAssociateHandle( + _In_ CXPLAT_EVENTQ* queue, + _In_ HANDLE fileHandle ) { - KeSetEvent(queue, IO_NO_INCREMENT, FALSE); + // TODO: implement + UNREFERENCED_PARAMETER(queue); + UNREFERENCED_PARAMETER(fileHandle); return TRUE; } -#define CxPlatEventQEnqueue(queue, sqe) _CxPlatEventQEnqueue(queue) +inline +BOOLEAN +CxPlatEventQEnqueue( + _In_ CXPLAT_EVENTQ* queue, + _In_ CXPLAT_SQE* sqe + ) +{ + return + NT_SUCCESS( + IoSetIoCompletionEx( + queue->IoCompletion, + 0, + sqe, + STATUS_SUCCESS, + 0, + FALSE, + sqe->MiniPacket)); +} +#define CXPLAT_EVENTQ_DEQUEUE_MAX 16 + +_IRQL_requires_max_(PASSIVE_LEVEL) inline uint32_t CxPlatEventQDequeue( _In_ CXPLAT_EVENTQ* queue, - _Out_ CXPLAT_CQE* events, - _In_ uint32_t count, + _Out_writes_to_(count, return) CXPLAT_CQE* events, + _In_range_(0, CXPLAT_EVENTQ_DEQUEUE_MAX) + uint32_t count, _In_ uint32_t wait_time // milliseconds ) { - UNREFERENCED_PARAMETER(count); - *events = NULL; - return STATUS_SUCCESS == _CxPlatEventWaitWithTimeout(queue, wait_time) ? 1 : 0; + NTSTATUS status; + ULONG entriesRemoved; + PLIST_ENTRY EntryPtrArray[CXPLAT_EVENTQ_DEQUEUE_MAX]; + if (wait_time == UINT32_MAX) { + status = + IoRemoveIoCompletion( + queue->IoCompletion, + events, + EntryPtrArray, + count, + &entriesRemoved, + KernelMode, + NULL, + FALSE); + } else { + LARGE_INTEGER timeout; + timeout.QuadPart = -10000LL * wait_time; + status = + IoRemoveIoCompletion( + queue->IoCompletion, + events, + EntryPtrArray, + count, + &entriesRemoved, + KernelMode, + &timeout, + FALSE); + } + return NT_SUCCESS(status) ? entriesRemoved : 0; } +_IRQL_requires_max_(PASSIVE_LEVEL) inline void CxPlatEventQReturn( @@ -533,14 +739,64 @@ CxPlatEventQReturn( } inline -void* -CxPlatCqeUserData( +BOOLEAN +CxPlatSqeInitialize( + _In_ CXPLAT_EVENTQ* queue, + _In_ CXPLAT_EVENT_COMPLETION_HANDLER completion, + _Out_ CXPLAT_SQE* sqe + ) +{ + UNREFERENCED_PARAMETER(queue); + sqe->Completion = completion; + sqe->MiniPacket = + IoAllocateMiniCompletionPacket( + IoMiniPacketCallbackRoutineNoOp, + NULL); + return sqe->MiniPacket ? TRUE : FALSE; +} + +inline +void +CxPlatSqeInitializeEx( + _In_ CXPLAT_EVENT_COMPLETION_HANDLER completion, + _Out_ CXPLAT_SQE* sqe + ) +{ + sqe->Completion = completion; + sqe->MiniPacket = + IoAllocateMiniCompletionPacket( + IoMiniPacketCallbackRoutineNoOp, + NULL); +} + +inline +void +CxPlatSqeCleanup( + _In_ CXPLAT_EVENTQ* queue, + _In_ CXPLAT_SQE* sqe + ) +{ + UNREFERENCED_PARAMETER(queue); + IoFreeMiniCompletionPacket(sqe->MiniPacket); +} + +inline +CXPLAT_SQE* +CxPlatCqeGetSqe( _In_ const CXPLAT_CQE* cqe ) { - return *cqe; + return (CXPLAT_SQE*)cqe->ApcContext; } +typedef struct DATAPATH_SQE DATAPATH_SQE; + +void +CxPlatDatapathSqeInitialize( + _Out_ DATAPATH_SQE* DatapathSqe, + _In_ uint32_t CqeType + ); + // // Time Measurement Interfaces // @@ -881,6 +1137,23 @@ typedef ULONG_PTR CXPLAT_THREAD_ID; // Processor Count and Index // +typedef struct CXPLAT_PROCESSOR_INFO { + uint16_t Group; // The group number this processor is a part of + uint8_t Index; // Index in the current group + uint8_t PADDING; // Here to align with PROCESSOR_NUMBER struct +} CXPLAT_PROCESSOR_INFO; + +CXPLAT_STATIC_ASSERT(sizeof(CXPLAT_PROCESSOR_INFO) == sizeof(PROCESSOR_NUMBER), "Size check"); + +typedef struct CXPLAT_PROCESSOR_GROUP_INFO { + KAFFINITY Mask; // Bit mask of active processors in the group + uint32_t Count; // Count of active processors in the group + uint32_t Offset; // Base process index offset this group starts at +} CXPLAT_PROCESSOR_GROUP_INFO; + +extern CXPLAT_PROCESSOR_INFO* CxPlatProcessorInfo; +extern CXPLAT_PROCESSOR_GROUP_INFO* CxPlatProcessorGroupInfo; + extern uint32_t CxPlatProcessorCount; #define CxPlatProcCount() CxPlatProcessorCount #define CxPlatProcCurrentNumber() (KeGetCurrentProcessorIndex() % CxPlatProcessorCount) diff --git a/src/inc/quic_platform_winuser.h b/src/inc/quic_platform_winuser.h index d04347ad76..d223bfba28 100644 --- a/src/inc/quic_platform_winuser.h +++ b/src/inc/quic_platform_winuser.h @@ -259,10 +259,23 @@ InterlockedFetchAndSetBoolean( _IRQL_requires_max_(PASSIVE_LEVEL) inline void -CxPlatCloseHandle(_Pre_notnull_ HANDLE Handle) { +CxPlatCloseHandle( + _Pre_notnull_ HANDLE Handle + ) +{ CloseHandle(Handle); } +_IRQL_requires_max_(PASSIVE_LEVEL) +inline +void +CxPlatCancelIo( + _Pre_notnull_ HANDLE Handle + ) +{ + CancelIoEx(Handle, NULL); +} + // // Allocation/Memory Interfaces // @@ -776,11 +789,13 @@ CxPlatEventQEnqueueEx( // Windows specific extension return PostQueuedCompletionStatus(*queue, num_bytes, 0, &sqe->Overlapped) != 0; } +#define CXPLAT_EVENTQ_DEQUEUE_MAX 16 + inline uint32_t CxPlatEventQDequeue( _In_ CXPLAT_EVENTQ* queue, - _Out_ CXPLAT_CQE* events, + _Out_writes_to_(count, return) CXPLAT_CQE* events, _In_ uint32_t count, _In_ uint32_t wait_time // milliseconds ) diff --git a/src/perf/bin/secnetperfdriver.kernel.vcxproj b/src/perf/bin/secnetperfdriver.kernel.vcxproj index ea84b40cd5..38c1bbe668 100644 --- a/src/perf/bin/secnetperfdriver.kernel.vcxproj +++ b/src/perf/bin/secnetperfdriver.kernel.vcxproj @@ -76,7 +76,14 @@ - ..\lib;..\..;..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\lib; + ..\..; + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir); + %(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/perf/bin/secnetperfdriverpriv.kernel.vcxproj b/src/perf/bin/secnetperfdriverpriv.kernel.vcxproj index bc0d6443a6..1e3477b5d7 100644 --- a/src/perf/bin/secnetperfdriverpriv.kernel.vcxproj +++ b/src/perf/bin/secnetperfdriverpriv.kernel.vcxproj @@ -76,7 +76,15 @@ - ..\lib;..\..;..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\lib; + ..\..; + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir); + %(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/perf/lib/perflib.kernel.vcxproj b/src/perf/lib/perflib.kernel.vcxproj index d642d9d43e..60ca395a40 100644 --- a/src/perf/lib/perflib.kernel.vcxproj +++ b/src/perf/lib/perflib.kernel.vcxproj @@ -68,7 +68,13 @@ - ..\;..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\; + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir); + %(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/platform/CMakeLists.txt b/src/platform/CMakeLists.txt index 61c898a8af..473822bd9c 100644 --- a/src/platform/CMakeLists.txt +++ b/src/platform/CMakeLists.txt @@ -20,14 +20,14 @@ if("${CX_PLATFORM}" STREQUAL "windows") ${SYSTEM_PROCESSOR} STREQUAL "arm64ec") set(SOURCES ${SOURCES} datapath_raw_dummy.c) else() - set(SOURCES ${SOURCES} datapath_raw.c datapath_raw_win.c datapath_raw_socket.c datapath_raw_socket_win.c datapath_raw_xdp_win.c) + set(SOURCES ${SOURCES} datapath_raw.c datapath_raw_win.c datapath_raw_socket.c datapath_raw_socket_win.c datapath_raw_socket_common.c datapath_raw_xdp_winuser.c) endif() else() set(SOURCES ${SOURCES} inline.c platform_posix.c storage_posix.c cgroup.c datapath_unix.c) if(CX_PLATFORM STREQUAL "linux" AND NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(SOURCES ${SOURCES} datapath_linux.c datapath_epoll.c) if (QUIC_LINUX_XDP_ENABLED) - set(SOURCES ${SOURCES} datapath_xplat.c datapath_raw.c datapath_raw_linux.c datapath_raw_socket.c datapath_raw_socket_linux.c datapath_raw_xdp_linux.c) + set(SOURCES ${SOURCES} datapath_xplat.c datapath_raw.c datapath_raw_linux.c datapath_raw_socket.c datapath_raw_socket_common.c datapath_raw_socket_linux.c datapath_raw_xdp_linux.c) else() set(SOURCES ${SOURCES} datapath_xplat.c datapath_raw_dummy.c) endif() diff --git a/src/platform/datapath_raw.h b/src/platform/datapath_raw.h index c980eee3e9..c9a2c6bc11 100644 --- a/src/platform/datapath_raw.h +++ b/src/platform/datapath_raw.h @@ -393,23 +393,60 @@ CxPlatFramingWriteHeaders( #pragma pack(push) #pragma pack(1) -typedef struct IPV6_EXTENSION { +typedef struct RAW_ETHERNET_HEADER { + uint8_t Destination[6]; + uint8_t Source[6]; + uint16_t Type; + uint8_t Data[0]; +} RAW_ETHERNET_HEADER; + +typedef struct RAW_IPV4_HEADER { + uint8_t VersionAndHeaderLength; + union { + uint8_t TypeOfServiceAndEcnField; + struct { + uint8_t EcnField : 2; + uint8_t TypeOfService : 6; + }; + }; + uint16_t TotalLength; + uint16_t Identification; + uint16_t FlagsAndFragmentOffset; + uint8_t TimeToLive; + uint8_t Protocol; + uint16_t HeaderChecksum; + uint8_t Source[4]; + uint8_t Destination[4]; + uint8_t Data[0]; +} RAW_IPV4_HEADER; + +typedef struct RAW_IPV6_HEADER { + uint32_t VersionClassEcnFlow; + uint16_t PayloadLength; + uint8_t NextHeader; + uint8_t HopLimit; + uint8_t Source[16]; + uint8_t Destination[16]; + uint8_t Data[0]; +} RAW_IPV6_HEADER; + +typedef struct RAW_IPV6_EXTENSION { uint8_t NextHeader; uint8_t Length; uint16_t Reserved0; uint32_t Reserved1; uint8_t Data[0]; -} IPV6_EXTENSION; +} RAW_IPV6_EXTENSION; -typedef struct UDP_HEADER { +typedef struct RAW_UDP_HEADER { uint16_t SourcePort; uint16_t DestinationPort; uint16_t Length; uint16_t Checksum; uint8_t Data[0]; -} UDP_HEADER; +} RAW_UDP_HEADER; -typedef struct TCP_HEADER { +typedef struct RAW_TCP_HEADER { uint16_t SourcePort; uint16_t DestinationPort; uint32_t SequenceNumber; @@ -420,7 +457,7 @@ typedef struct TCP_HEADER { uint16_t Window; uint16_t Checksum; uint16_t UrgentPointer; -} TCP_HEADER; +} RAW_TCP_HEADER; #pragma pack(pop) @@ -437,53 +474,12 @@ typedef struct TCP_HEADER { #define TH_ECE 0x40 #define TH_CWR 0x80 -#define IPV4_VERSION 4 -#define IPV4_VERSION_BYTE (IPV4_VERSION << 4) +#define RAW_IPV4_VERSION 4 +#define RAW_IPV6_VERSION 6 +#define RAW_IPV4_VERSION_BYTE (RAW_IPV4_VERSION << 4) +#define RAW_IPV4_DEFAULT_VERHLEN ((RAW_IPV4_VERSION_BYTE) | (sizeof(RAW_IPV4_HEADER) / sizeof(uint32_t))) -#define IP_DEFAULT_HOP_LIMIT 128 +#define RAW_IP_DEFAULT_HOP_LIMIT 128 -#ifndef _KERNEL_MODE -typedef struct ETHERNET_HEADER { - uint8_t Destination[6]; - uint8_t Source[6]; - uint16_t Type; - uint8_t Data[0]; -} ETHERNET_HEADER; - -typedef struct IPV4_HEADER { - uint8_t VersionAndHeaderLength; - union { - uint8_t TypeOfServiceAndEcnField; - struct { - uint8_t EcnField : 2; - uint8_t TypeOfService : 6; - }; - }; - uint16_t TotalLength; - uint16_t Identification; - uint16_t FlagsAndFragmentOffset; - uint8_t TimeToLive; - uint8_t Protocol; - uint16_t HeaderChecksum; - uint8_t Source[4]; - uint8_t Destination[4]; - uint8_t Data[0]; -} IPV4_HEADER; - -typedef struct IPV6_HEADER { - uint32_t VersionClassEcnFlow; - uint16_t PayloadLength; - uint8_t NextHeader; - uint8_t HopLimit; - uint8_t Source[16]; - uint8_t Destination[16]; - uint8_t Data[0]; -} IPV6_HEADER; - -#define IPV6_VERSION 6 -#define IPV4_DEFAULT_VERHLEN ((IPV4_VERSION_BYTE) | (sizeof(IPV4_HEADER) / sizeof(uint32_t))) - -#define ETHERNET_TYPE_IPV4 0x0008 -#define ETHERNET_TYPE_IPV6 0xdd86 - -#endif +#define RAW_ETHERNET_TYPE_IPV4 0x0008 +#define RAW_ETHERNET_TYPE_IPV6 0xdd86 diff --git a/src/platform/datapath_raw_socket.c b/src/platform/datapath_raw_socket.c index 9641ea9109..f29a57e698 100644 --- a/src/platform/datapath_raw_socket.c +++ b/src/platform/datapath_raw_socket.c @@ -14,20 +14,6 @@ #include "datapath_raw_socket.c.clog.h" #endif -#if defined(CX_PLATFORM_LINUX) || defined(CX_PLATFORM_DARWIN) -#define CxPlatSocketError() errno -#define CxPlatCloseSocket(s) close(s) -#define CxPlatQuicErrorFromSocketError(e) (QUIC_STATUS)e -#define CxPlatAddressLengthType uint32_t -#elif defined(_WIN32) || defined(_KERNEL_MODE) -#define CxPlatSocketError() WSAGetLastError() -#define CxPlatCloseSocket(s) closesocket(s) -#define CxPlatQuicErrorFromSocketError(e) HRESULT_FROM_WIN32(e) -#define CxPlatAddressLengthType int -#else -#error unsupported platform -#endif - #pragma warning(disable:4116) // unnamed type definition in parentheses #pragma warning(disable:4100) // unreferenced formal parameter @@ -73,29 +59,6 @@ CxPlatGetSocket( return Socket; } -void -CxPlatRemoveSocket( - _In_ CXPLAT_SOCKET_POOL* Pool, - _In_ CXPLAT_SOCKET_RAW* Socket - ) -{ - CxPlatRwLockAcquireExclusive(&Pool->Lock); - CxPlatHashtableRemove(&Pool->Sockets, &Socket->Entry, NULL); - - if (Socket->AuxSocket != INVALID_SOCKET && - CxPlatCloseSocket(Socket->AuxSocket) == SOCKET_ERROR) { - int Error = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - Error, - "closesocket"); - } - - CxPlatRwLockReleaseExclusive(&Pool->Lock); -} - void RawResolveRouteComplete( _In_ void* Context, @@ -152,11 +115,11 @@ CxPlatDpRawParseUdp( _In_ const CXPLAT_DATAPATH* Datapath, _Inout_ CXPLAT_RECV_DATA* Packet, _In_reads_bytes_(Length) - const UDP_HEADER* Udp, + const RAW_UDP_HEADER* Udp, _In_ uint16_t Length ) { - if (Length < sizeof(UDP_HEADER)) { + if (Length < sizeof(RAW_UDP_HEADER)) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -182,7 +145,7 @@ CxPlatDpRawParseUdp( Packet->Route->LocalAddress.Ipv4.sin_port = Udp->DestinationPort; Packet->Buffer = (uint8_t*)Udp->Data; - Packet->BufferLength = QuicNetByteSwapShort(Udp->Length) - sizeof(UDP_HEADER); + Packet->BufferLength = QuicNetByteSwapShort(Udp->Length) - sizeof(RAW_UDP_HEADER); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -192,12 +155,12 @@ CxPlatDpRawParseTcp( _In_ const CXPLAT_DATAPATH* Datapath, _Inout_ CXPLAT_RECV_DATA* Packet, _In_reads_bytes_(Length) - const TCP_HEADER* Tcp, + const RAW_TCP_HEADER* Tcp, _In_ uint16_t Length ) { uint16_t HeaderLength; - if (Length < sizeof(TCP_HEADER)) { + if (Length < sizeof(RAW_TCP_HEADER)) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -262,11 +225,11 @@ CxPlatDpRawParseIPv4( _In_ const CXPLAT_DATAPATH* Datapath, _Inout_ CXPLAT_RECV_DATA* Packet, _In_reads_bytes_(Length) - const IPV4_HEADER* IP, + const RAW_IPV4_HEADER* IP, _In_ uint16_t Length ) { - if (Length < sizeof(IPV4_HEADER)) { + if (Length < sizeof(RAW_IPV4_HEADER)) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -276,7 +239,7 @@ CxPlatDpRawParseIPv4( return; } - if (IP->VersionAndHeaderLength != IPV4_DEFAULT_VERHLEN) { + if (IP->VersionAndHeaderLength != RAW_IPV4_DEFAULT_VERHLEN) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -305,9 +268,9 @@ CxPlatDpRawParseIPv4( CxPlatCopyMemory(&Packet->Route->LocalAddress.Ipv4.sin_addr, IP->Destination, sizeof(IP->Destination)); if (IP->Protocol == IPPROTO_UDP) { - CxPlatDpRawParseUdp(Datapath, Packet, (UDP_HEADER*)IP->Data, IPTotalLength - sizeof(IPV4_HEADER)); + CxPlatDpRawParseUdp(Datapath, Packet, (RAW_UDP_HEADER*)IP->Data, IPTotalLength - sizeof(RAW_IPV4_HEADER)); } else if (IP->Protocol == IPPROTO_TCP) { - CxPlatDpRawParseTcp(Datapath, Packet, (TCP_HEADER*)IP->Data, IPTotalLength - sizeof(IPV4_HEADER)); + CxPlatDpRawParseTcp(Datapath, Packet, (RAW_TCP_HEADER*)IP->Data, IPTotalLength - sizeof(RAW_IPV4_HEADER)); } else { QuicTraceEvent( DatapathErrorStatus, @@ -325,12 +288,12 @@ CxPlatDpRawParseIPv6( _In_ const CXPLAT_DATAPATH* Datapath, _Inout_ CXPLAT_RECV_DATA* Packet, _In_reads_bytes_(Length) - const IPV6_HEADER* IP, + const RAW_IPV6_HEADER* IP, _In_ uint16_t Length ) { - if (Length < sizeof(IPV6_HEADER)) { + if (Length < sizeof(RAW_IPV6_HEADER)) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -341,7 +304,7 @@ CxPlatDpRawParseIPv6( } uint16_t IPPayloadLength = CxPlatByteSwapUint16(IP->PayloadLength); - if (IPPayloadLength + sizeof(IPV6_HEADER) > Length) { + if (IPPayloadLength + sizeof(RAW_IPV6_HEADER) > Length) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -374,9 +337,9 @@ CxPlatDpRawParseIPv6( CxPlatCopyMemory(&Packet->Route->LocalAddress.Ipv6.sin6_addr, IP->Destination, sizeof(IP->Destination)); if (IP->NextHeader == IPPROTO_UDP) { - CxPlatDpRawParseUdp(Datapath, Packet, (UDP_HEADER*)IP->Data, IPPayloadLength); + CxPlatDpRawParseUdp(Datapath, Packet, (RAW_UDP_HEADER*)IP->Data, IPPayloadLength); } else if (IP->NextHeader == IPPROTO_TCP) { - CxPlatDpRawParseTcp(Datapath, Packet, (TCP_HEADER*)IP->Data, IPPayloadLength); + CxPlatDpRawParseTcp(Datapath, Packet, (RAW_TCP_HEADER*)IP->Data, IPPayloadLength); } else { QuicTraceEvent( DatapathErrorStatus, @@ -407,7 +370,7 @@ CxPlatDpRawParseEthernet( _In_ uint16_t Length ) { - if (Length < sizeof(ETHERNET_HEADER)) { + if (Length < sizeof(RAW_ETHERNET_HEADER)) { QuicTraceEvent( DatapathErrorStatus, "[data][%p] ERROR, %u, %s.", @@ -417,9 +380,9 @@ CxPlatDpRawParseEthernet( return; } - Length -= sizeof(ETHERNET_HEADER); + Length -= sizeof(RAW_ETHERNET_HEADER); - const ETHERNET_HEADER* Ethernet = (const ETHERNET_HEADER*)Payload; + const RAW_ETHERNET_HEADER* Ethernet = (const RAW_ETHERNET_HEADER*)Payload; if (IsEthernetBroadcast(Ethernet->Destination) || IsEthernetMulticast(Ethernet->Destination)) { QuicTraceEvent( @@ -435,10 +398,10 @@ CxPlatDpRawParseEthernet( CxPlatCopyMemory(&Packet->Route->NextHopLinkLayerAddress, Ethernet->Source, sizeof(Ethernet->Source)); uint16_t EthernetType = Ethernet->Type; - if (EthernetType == ETHERNET_TYPE_IPV4) { - CxPlatDpRawParseIPv4(Datapath, Packet, (IPV4_HEADER*)Ethernet->Data, Length); - } else if (EthernetType == ETHERNET_TYPE_IPV6) { - CxPlatDpRawParseIPv6(Datapath, Packet, (IPV6_HEADER*)Ethernet->Data, Length); + if (EthernetType == RAW_ETHERNET_TYPE_IPV4) { + CxPlatDpRawParseIPv4(Datapath, Packet, (RAW_IPV4_HEADER*)Ethernet->Data, Length); + } else if (EthernetType == RAW_ETHERNET_TYPE_IPV6) { + CxPlatDpRawParseIPv6(Datapath, Packet, (RAW_IPV6_HEADER*)Ethernet->Data, Length); } else { QuicTraceEvent( DatapathErrorStatus, @@ -457,10 +420,10 @@ CxPlatDpRawCalculateHeaderBackFill( ) { HEADER_BACKFILL HeaderBackFill; - HeaderBackFill.TransportLayer = UseTcp ? sizeof(TCP_HEADER) : sizeof(UDP_HEADER); + HeaderBackFill.TransportLayer = UseTcp ? sizeof(RAW_TCP_HEADER) : sizeof(RAW_UDP_HEADER); HeaderBackFill.NetworkLayer = - Family == QUIC_ADDRESS_FAMILY_INET ? sizeof(IPV4_HEADER) : sizeof(IPV6_HEADER); - HeaderBackFill.LinkLayer = sizeof(ETHERNET_HEADER); + Family == QUIC_ADDRESS_FAMILY_INET ? sizeof(RAW_IPV4_HEADER) : sizeof(RAW_IPV6_HEADER); + HeaderBackFill.LinkLayer = sizeof(RAW_ETHERNET_HEADER); HeaderBackFill.AllLayer = HeaderBackFill.TransportLayer + HeaderBackFill.NetworkLayer + HeaderBackFill.LinkLayer; return HeaderBackFill; @@ -555,7 +518,7 @@ CxPlatDpRawSocketAckFin( CXPLAT_DBG_ASSERT(Route->State == RouteResolved); CXPLAT_DBG_ASSERT(Route->Queue != NULL); const CXPLAT_INTERFACE* Interface = CxPlatDpRawGetInterfaceFromQueue(Route->Queue); - TCP_HEADER* ReceivedTcpHeader = (TCP_HEADER*)(Packet->Buffer - Packet->ReservedEx); + RAW_TCP_HEADER* ReceivedTcpHeader = (RAW_TCP_HEADER*)(Packet->Buffer - Packet->ReservedEx); CxPlatFramingWriteHeaders( Socket, Route, &SendData->Buffer, SendData->ECN, SendData->DSCP, @@ -587,7 +550,7 @@ CxPlatDpRawSocketAckSyn( CXPLAT_DBG_ASSERT(Route->State == RouteResolved); CXPLAT_DBG_ASSERT(Route->Queue != NULL); const CXPLAT_INTERFACE* Interface = CxPlatDpRawGetInterfaceFromQueue(Route->Queue); - TCP_HEADER* ReceivedTcpHeader = (TCP_HEADER*)(Packet->Buffer - Packet->ReservedEx); + RAW_TCP_HEADER* ReceivedTcpHeader = (RAW_TCP_HEADER*)(Packet->Buffer - Packet->ReservedEx); QuicTraceEvent( DatapathSendTcpControl, @@ -703,9 +666,9 @@ CxPlatFramingWriteHeaders( uint8_t* Transport; uint16_t TransportLength; uint8_t TransportProtocol; - TCP_HEADER* TCP = NULL; - UDP_HEADER* UDP = NULL; - ETHERNET_HEADER* Ethernet; + RAW_TCP_HEADER* TCP = NULL; + RAW_UDP_HEADER* UDP = NULL; + RAW_ETHERNET_HEADER* Ethernet; uint16_t EthType; uint16_t IpHeaderLen; QUIC_ADDRESS_FAMILY Family = QuicAddrGetFamily(&Route->RemoteAddress); @@ -717,32 +680,32 @@ CxPlatFramingWriteHeaders( // // Fill TCP header. // - TCP = (TCP_HEADER*)(Buffer->Buffer - sizeof(TCP_HEADER)); + TCP = (RAW_TCP_HEADER*)(Buffer->Buffer - sizeof(RAW_TCP_HEADER)); TCP->DestinationPort = Route->RemoteAddress.Ipv4.sin_port; TCP->SourcePort = Route->LocalAddress.Ipv4.sin_port; TCP->Window = 0xFFFF; TCP->X2 = 0; TCP->Checksum = 0; TCP->UrgentPointer = 0; - TCP->HeaderLength = sizeof(TCP_HEADER) / sizeof(uint32_t); + TCP->HeaderLength = sizeof(RAW_TCP_HEADER) / sizeof(uint32_t); TCP->SequenceNumber = TcpSeqNum; TCP->AckNumber = TcpAckNum; TCP->Flags = TcpFlags; Transport = (uint8_t*)TCP; - TransportLength = sizeof(TCP_HEADER); + TransportLength = sizeof(RAW_TCP_HEADER); TransportProtocol = IPPROTO_TCP; } else { // // Fill UDP header. // - UDP = (UDP_HEADER*)(Buffer->Buffer - sizeof(UDP_HEADER)); + UDP = (RAW_UDP_HEADER*)(Buffer->Buffer - sizeof(RAW_UDP_HEADER)); UDP->DestinationPort = Route->RemoteAddress.Ipv4.sin_port; UDP->SourcePort = Route->LocalAddress.Ipv4.sin_port; - UDP->Length = QuicNetByteSwapShort((uint16_t)Buffer->Length + sizeof(UDP_HEADER)); + UDP->Length = QuicNetByteSwapShort((uint16_t)Buffer->Length + sizeof(RAW_UDP_HEADER)); UDP->Checksum = 0; Transport = (uint8_t*)UDP; - TransportLength = sizeof(UDP_HEADER); + TransportLength = sizeof(RAW_UDP_HEADER); TransportProtocol = IPPROTO_UDP; } @@ -750,22 +713,22 @@ CxPlatFramingWriteHeaders( // Fill IPv4/IPv6 header. // if (Family == QUIC_ADDRESS_FAMILY_INET) { - IPV4_HEADER* IPv4 = (IPV4_HEADER*)(Transport - sizeof(IPV4_HEADER)); - IPv4->VersionAndHeaderLength = IPV4_DEFAULT_VERHLEN; + RAW_IPV4_HEADER* IPv4 = (RAW_IPV4_HEADER*)(Transport - sizeof(RAW_IPV4_HEADER)); + IPv4->VersionAndHeaderLength = RAW_IPV4_DEFAULT_VERHLEN; IPv4->TypeOfService = DSCP; IPv4->EcnField = ECN; - IPv4->TotalLength = htons(sizeof(IPV4_HEADER) + TransportLength + (uint16_t)Buffer->Length); + IPv4->TotalLength = htons(sizeof(RAW_IPV4_HEADER) + TransportLength + (uint16_t)Buffer->Length); IPv4->Identification = 0; IPv4->FlagsAndFragmentOffset = 0; - IPv4->TimeToLive = IP_DEFAULT_HOP_LIMIT; + IPv4->TimeToLive = RAW_IP_DEFAULT_HOP_LIMIT; IPv4->Protocol = TransportProtocol; IPv4->HeaderChecksum = 0; CxPlatCopyMemory(IPv4->Source, &Route->LocalAddress.Ipv4.sin_addr, sizeof(Route->LocalAddress.Ipv4.sin_addr)); CxPlatCopyMemory(IPv4->Destination, &Route->RemoteAddress.Ipv4.sin_addr, sizeof(Route->RemoteAddress.Ipv4.sin_addr)); - IPv4->HeaderChecksum = SkipNetworkLayerXsum ? 0 : ~CxPlatFramingChecksum((uint8_t*)IPv4, sizeof(IPV4_HEADER), 0); - EthType = ETHERNET_TYPE_IPV4; - Ethernet = (ETHERNET_HEADER*)(((uint8_t*)IPv4) - sizeof(ETHERNET_HEADER)); - IpHeaderLen = sizeof(IPV4_HEADER); + IPv4->HeaderChecksum = SkipNetworkLayerXsum ? 0 : ~CxPlatFramingChecksum((uint8_t*)IPv4, sizeof(RAW_IPV4_HEADER), 0); + EthType = RAW_ETHERNET_TYPE_IPV4; + Ethernet = (RAW_ETHERNET_HEADER*)(((uint8_t*)IPv4) - sizeof(RAW_ETHERNET_HEADER)); + IpHeaderLen = sizeof(RAW_IPV4_HEADER); if (!SkipTransportLayerXsum) { if (Socket->UseTcp) { TCP->Checksum = @@ -773,18 +736,18 @@ CxPlatFramingWriteHeaders( IPv4->Source, IPv4->Destination, sizeof(Route->LocalAddress.Ipv4.sin_addr), IPPROTO_TCP, - (uint8_t*)TCP, sizeof(TCP_HEADER) + Buffer->Length); + (uint8_t*)TCP, sizeof(RAW_TCP_HEADER) + Buffer->Length); } else { UDP->Checksum = CxPlatFramingTransportChecksum( IPv4->Source, IPv4->Destination, sizeof(Route->LocalAddress.Ipv4.sin_addr), IPPROTO_UDP, - (uint8_t*)UDP, sizeof(UDP_HEADER) + Buffer->Length); + (uint8_t*)UDP, sizeof(RAW_UDP_HEADER) + Buffer->Length); } } } else { - IPV6_HEADER* IPv6 = (IPV6_HEADER*)(Transport - sizeof(IPV6_HEADER)); + RAW_IPV6_HEADER* IPv6 = (RAW_IPV6_HEADER*)(Transport - sizeof(RAW_IPV6_HEADER)); // // IPv6 Version, Traffic Class, ECN Field and Flow Label fields in host // byte order. @@ -799,20 +762,20 @@ CxPlatFramingWriteHeaders( uint32_t Value; } VersionClassEcnFlow = {0}; - VersionClassEcnFlow.Version = IPV6_VERSION; + VersionClassEcnFlow.Version = RAW_IPV6_VERSION; VersionClassEcnFlow.Class = DSCP; VersionClassEcnFlow.EcnField = ECN; VersionClassEcnFlow.Flow = (uint32_t)(uintptr_t)Socket; IPv6->VersionClassEcnFlow = CxPlatByteSwapUint32(VersionClassEcnFlow.Value); IPv6->PayloadLength = htons(TransportLength + (uint16_t)Buffer->Length); - IPv6->HopLimit = IP_DEFAULT_HOP_LIMIT; + IPv6->HopLimit = RAW_IP_DEFAULT_HOP_LIMIT; IPv6->NextHeader = TransportProtocol; CxPlatCopyMemory(IPv6->Source, &Route->LocalAddress.Ipv6.sin6_addr, sizeof(Route->LocalAddress.Ipv6.sin6_addr)); CxPlatCopyMemory(IPv6->Destination, &Route->RemoteAddress.Ipv6.sin6_addr, sizeof(Route->RemoteAddress.Ipv6.sin6_addr)); - EthType = ETHERNET_TYPE_IPV6; - Ethernet = (ETHERNET_HEADER*)(((uint8_t*)IPv6) - sizeof(ETHERNET_HEADER)); - IpHeaderLen = sizeof(IPV6_HEADER); + EthType = RAW_ETHERNET_TYPE_IPV6; + Ethernet = (RAW_ETHERNET_HEADER*)(((uint8_t*)IPv6) - sizeof(RAW_ETHERNET_HEADER)); + IpHeaderLen = sizeof(RAW_IPV6_HEADER); if (!SkipTransportLayerXsum) { if (Socket->UseTcp) { TCP->Checksum = @@ -820,14 +783,14 @@ CxPlatFramingWriteHeaders( IPv6->Source, IPv6->Destination, sizeof(Route->LocalAddress.Ipv6.sin6_addr), IPPROTO_TCP, - (uint8_t*)TCP, sizeof(TCP_HEADER) + Buffer->Length); + (uint8_t*)TCP, sizeof(RAW_TCP_HEADER) + Buffer->Length); } else { UDP->Checksum = CxPlatFramingTransportChecksum( IPv6->Source, IPv6->Destination, sizeof(Route->LocalAddress.Ipv6.sin6_addr), IPPROTO_UDP, - (uint8_t*)UDP, sizeof(UDP_HEADER) + Buffer->Length); + (uint8_t*)UDP, sizeof(RAW_UDP_HEADER) + Buffer->Length); UDP->Checksum = UDP->Checksum != 0 ? UDP->Checksum : ~0; } } @@ -840,303 +803,6 @@ CxPlatFramingWriteHeaders( CxPlatCopyMemory(Ethernet->Destination, Route->NextHopLinkLayerAddress, sizeof(Route->NextHopLinkLayerAddress)); CxPlatCopyMemory(Ethernet->Source, Route->LocalLinkLayerAddress, sizeof(Route->LocalLinkLayerAddress)); - Buffer->Length += TransportLength + IpHeaderLen + sizeof(ETHERNET_HEADER); - Buffer->Buffer -= TransportLength + IpHeaderLen + sizeof(ETHERNET_HEADER); -} - -QUIC_STATUS -CxPlatTryAddSocket( - _In_ CXPLAT_SOCKET_POOL* Pool, - _In_ CXPLAT_SOCKET_RAW* Socket - ) -{ - QUIC_STATUS Status = QUIC_STATUS_SUCCESS; - int Result; - CXPLAT_HASHTABLE_LOOKUP_CONTEXT Context; - CXPLAT_HASHTABLE_ENTRY* Entry; - int Option; - QUIC_ADDR MappedAddress = {0}; - SOCKET TempUdpSocket = INVALID_SOCKET; - CxPlatAddressLengthType AssignedLocalAddressLength; - - // - // Get (and reserve) a transport layer port from the OS networking stack by - // binding an auxiliary (dual stack) socket. - // - - if (Socket->UseTcp) { - Socket->AuxSocket = - socket( - AF_INET6, - SOCK_STREAM, - IPPROTO_TCP); - if (Socket->AuxSocket == INVALID_SOCKET) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "socket"); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - Option = FALSE; - Result = - setsockopt( - Socket->AuxSocket, - IPPROTO_IPV6, - IPV6_V6ONLY, - (char*)&Option, - sizeof(Option)); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "Set IPV6_V6ONLY"); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - if (Socket->CibirIdLength) { - Option = TRUE; - Result = - setsockopt( - Socket->AuxSocket, - SOL_SOCKET, - SO_REUSEADDR, - (char*)&Option, - sizeof(Option)); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "Set SO_REUSEADDR"); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - } - - CxPlatConvertToMappedV6(&Socket->LocalAddress, &MappedAddress); -#if QUIC_ADDRESS_FAMILY_INET6 != AF_INET6 - if (MappedAddress.Ipv6.sin6_family == QUIC_ADDRESS_FAMILY_INET6) { - MappedAddress.Ipv6.sin6_family = AF_INET6; - } -#endif - } - - CxPlatRwLockAcquireExclusive(&Pool->Lock); - - if (Socket->UseTcp) { - QUIC_ADDR_STR LocalAddressString = {0}; - QuicAddrToString(&MappedAddress, &LocalAddressString); - QuicTraceLogVerbose( - DatapathTcpAuxBinding, - "[data][%p] Binding TCP socket to %s", - Socket, - LocalAddressString.Address); - Result = - bind( - Socket->AuxSocket, - (struct sockaddr*)&MappedAddress, - sizeof(MappedAddress)); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "bind"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - if (Socket->Connected) { - CxPlatZeroMemory(&MappedAddress, sizeof(MappedAddress)); - CxPlatConvertToMappedV6(&Socket->RemoteAddress, &MappedAddress); - -#if QUIC_ADDRESS_FAMILY_INET6 != AF_INET6 - if (MappedAddress.Ipv6.sin6_family == QUIC_ADDRESS_FAMILY_INET6) { - MappedAddress.Ipv6.sin6_family = AF_INET6; - } -#endif - // - // Create a temporary UDP socket bound to a wildcard port - // and connect this socket to the remote address. - // By doing this, the OS will select a local address for us. - // - uint16_t LocalPortChosen = 0; - QUIC_ADDR TempLocalAddress = {0}; - AssignedLocalAddressLength = sizeof(TempLocalAddress); - Result = - getsockname( - Socket->AuxSocket, - (struct sockaddr*)&TempLocalAddress, - &AssignedLocalAddressLength); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "getsockname"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - LocalPortChosen = TempLocalAddress.Ipv4.sin_port; - TempUdpSocket = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP); - if (TempUdpSocket == INVALID_SOCKET) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "temp udp socket"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - Option = FALSE; - Result = - setsockopt( - TempUdpSocket, - IPPROTO_IPV6, - IPV6_V6ONLY, - (char*)&Option, - sizeof(Option)); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "Set IPV6_V6ONLY (temp udp socket)"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - CxPlatZeroMemory(&TempLocalAddress, sizeof(TempLocalAddress)); - CxPlatConvertToMappedV6(&Socket->LocalAddress, &TempLocalAddress); - TempLocalAddress.Ipv4.sin_port = 0; - Result = - bind( - TempUdpSocket, - (struct sockaddr*)&TempLocalAddress, - sizeof(TempLocalAddress)); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "bind (temp udp socket)"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - Result = - connect( - TempUdpSocket, - (struct sockaddr*)&MappedAddress, - sizeof(MappedAddress)); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "connect failed (temp udp socket)"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - - AssignedLocalAddressLength = sizeof(Socket->LocalAddress); - Result = - getsockname( - TempUdpSocket, - (struct sockaddr*)&Socket->LocalAddress, - &AssignedLocalAddressLength); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "getsockname (temp udp socket)"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - CxPlatConvertFromMappedV6(&Socket->LocalAddress, &Socket->LocalAddress); - Socket->LocalAddress.Ipv4.sin_port = LocalPortChosen; - CXPLAT_FRE_ASSERT(Socket->LocalAddress.Ipv4.sin_port != 0); - } else { - AssignedLocalAddressLength = sizeof(Socket->LocalAddress); - Result = - getsockname( - Socket->AuxSocket, - (struct sockaddr*)&Socket->LocalAddress, - &AssignedLocalAddressLength); - if (Result == SOCKET_ERROR) { - int WsaError = CxPlatSocketError(); - QuicTraceEvent( - DatapathErrorStatus, - "[data][%p] ERROR, %u, %s.", - Socket, - WsaError, - "getsockname"); - CxPlatRwLockReleaseExclusive(&Pool->Lock); - Status = CxPlatQuicErrorFromSocketError(WsaError); - goto Error; - } - CxPlatConvertFromMappedV6(&Socket->LocalAddress, &Socket->LocalAddress); - } - } - - Entry = CxPlatHashtableLookup(&Pool->Sockets, Socket->LocalAddress.Ipv4.sin_port, &Context); - while (Entry != NULL) { - CXPLAT_SOCKET_RAW* Temp = CXPLAT_CONTAINING_RECORD(Entry, CXPLAT_SOCKET_RAW, Entry); - if (CxPlatSocketCompare(Temp, &Socket->LocalAddress, &Socket->RemoteAddress)) { - Status = QUIC_STATUS_ADDRESS_IN_USE; - break; - } - Entry = CxPlatHashtableLookupNext(&Pool->Sockets, &Context); - } - if (QUIC_SUCCEEDED(Status)) { - CxPlatHashtableInsert(&Pool->Sockets, &Socket->Entry, Socket->LocalAddress.Ipv4.sin_port, &Context); - } - - CxPlatRwLockReleaseExclusive(&Pool->Lock); - -Error: - - if (QUIC_FAILED(Status) && Socket->AuxSocket != INVALID_SOCKET) { - CxPlatCloseSocket(Socket->AuxSocket); - } - - if (TempUdpSocket != INVALID_SOCKET) { - CxPlatCloseSocket(TempUdpSocket); - } - - return Status; + Buffer->Length += TransportLength + IpHeaderLen + sizeof(RAW_ETHERNET_HEADER); + Buffer->Buffer -= TransportLength + IpHeaderLen + sizeof(RAW_ETHERNET_HEADER); } diff --git a/src/platform/datapath_raw_socket_common.c b/src/platform/datapath_raw_socket_common.c new file mode 100644 index 0000000000..de4b5451c1 --- /dev/null +++ b/src/platform/datapath_raw_socket_common.c @@ -0,0 +1,405 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + QUIC raw datapath socket and IP framing abstractions + +--*/ + +#include "datapath_raw.h" +#ifdef QUIC_CLOG +#include "datapath_raw_socket_common.c.clog.h" +#endif + + +#if defined(CX_PLATFORM_LINUX) || defined(CX_PLATFORM_DARWIN) +#define CxPlatSocketError() errno +#define CxPlatCloseSocket(s) close(s) +#define CxPlatQuicErrorFromSocketError(e) (QUIC_STATUS)e +#define CxPlatAddressLengthType uint32_t +#elif defined(_WIN32) +#define CxPlatSocketError() WSAGetLastError() +#define CxPlatCloseSocket(s) closesocket(s) +#define CxPlatQuicErrorFromSocketError(e) HRESULT_FROM_WIN32(e) +#define CxPlatAddressLengthType int +#else +#error unsupported platform +#endif + +#pragma warning(disable:4116) // unnamed type definition in parentheses +#pragma warning(disable:4100) // unreferenced formal parameter + +#ifdef _KERNEL_MODE + +void +CxPlatRemoveSocket( + _In_ CXPLAT_SOCKET_POOL* Pool, + _In_ CXPLAT_SOCKET_RAW* Socket + ) +{ + CxPlatRwLockAcquireExclusive(&Pool->Lock); + CxPlatHashtableRemove(&Pool->Sockets, &Socket->Entry, NULL); + CxPlatRwLockReleaseExclusive(&Pool->Lock); +} + +QUIC_STATUS +CxPlatTryAddSocket( + _In_ CXPLAT_SOCKET_POOL* Pool, + _In_ CXPLAT_SOCKET_RAW* Socket + ) +{ + QUIC_STATUS Status = QUIC_STATUS_SUCCESS; + CXPLAT_HASHTABLE_LOOKUP_CONTEXT Context; + CXPLAT_HASHTABLE_ENTRY* Entry; + + // + // Get (and reserve) a transport layer port from the OS networking stack by + // binding an auxiliary (dual stack) socket. + // + + CxPlatRwLockAcquireExclusive(&Pool->Lock); + + Entry = CxPlatHashtableLookup(&Pool->Sockets, Socket->LocalAddress.Ipv4.sin_port, &Context); + while (Entry != NULL) { + CXPLAT_SOCKET_RAW* Temp = CXPLAT_CONTAINING_RECORD(Entry, CXPLAT_SOCKET_RAW, Entry); + if (CxPlatSocketCompare(Temp, &Socket->LocalAddress, &Socket->RemoteAddress)) { + Status = QUIC_STATUS_ADDRESS_IN_USE; + break; + } + Entry = CxPlatHashtableLookupNext(&Pool->Sockets, &Context); + } + if (QUIC_SUCCEEDED(Status)) { + CxPlatHashtableInsert(&Pool->Sockets, &Socket->Entry, Socket->LocalAddress.Ipv4.sin_port, &Context); + } + + CxPlatRwLockReleaseExclusive(&Pool->Lock); + + return Status; +} + +#else + +void +CxPlatRemoveSocket( + _In_ CXPLAT_SOCKET_POOL* Pool, + _In_ CXPLAT_SOCKET_RAW* Socket + ) +{ + CxPlatRwLockAcquireExclusive(&Pool->Lock); + CxPlatHashtableRemove(&Pool->Sockets, &Socket->Entry, NULL); + + if (Socket->AuxSocket != INVALID_SOCKET && + CxPlatCloseSocket(Socket->AuxSocket) == SOCKET_ERROR) { + int Error = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Error, + "closesocket"); + } + + CxPlatRwLockReleaseExclusive(&Pool->Lock); +} + +QUIC_STATUS +CxPlatTryAddSocket( + _In_ CXPLAT_SOCKET_POOL* Pool, + _In_ CXPLAT_SOCKET_RAW* Socket + ) +{ + QUIC_STATUS Status = QUIC_STATUS_SUCCESS; + int Result; + CXPLAT_HASHTABLE_LOOKUP_CONTEXT Context; + CXPLAT_HASHTABLE_ENTRY* Entry; + int Option; + QUIC_ADDR MappedAddress = {0}; + SOCKET TempUdpSocket = INVALID_SOCKET; + CxPlatAddressLengthType AssignedLocalAddressLength; + + // + // Get (and reserve) a transport layer port from the OS networking stack by + // binding an auxiliary (dual stack) socket. + // + + if (Socket->UseTcp) { + Socket->AuxSocket = + socket( + AF_INET6, + SOCK_STREAM, + IPPROTO_TCP); + if (Socket->AuxSocket == INVALID_SOCKET) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "socket"); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + Option = FALSE; + Result = + setsockopt( + Socket->AuxSocket, + IPPROTO_IPV6, + IPV6_V6ONLY, + (char*)&Option, + sizeof(Option)); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "Set IPV6_V6ONLY"); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + if (Socket->CibirIdLength) { + Option = TRUE; + Result = + setsockopt( + Socket->AuxSocket, + SOL_SOCKET, + SO_REUSEADDR, + (char*)&Option, + sizeof(Option)); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "Set SO_REUSEADDR"); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + } + + CxPlatConvertToMappedV6(&Socket->LocalAddress, &MappedAddress); +#if QUIC_ADDRESS_FAMILY_INET6 != AF_INET6 + if (MappedAddress.Ipv6.sin6_family == QUIC_ADDRESS_FAMILY_INET6) { + MappedAddress.Ipv6.sin6_family = AF_INET6; + } +#endif + } + + CxPlatRwLockAcquireExclusive(&Pool->Lock); + + if (Socket->UseTcp) { + QUIC_ADDR_STR LocalAddressString = {0}; + QuicAddrToString(&MappedAddress, &LocalAddressString); + QuicTraceLogVerbose( + DatapathTcpAuxBinding, + "[data][%p] Binding TCP socket to %s", + Socket, + LocalAddressString.Address); + Result = + bind( + Socket->AuxSocket, + (struct sockaddr*)&MappedAddress, + sizeof(MappedAddress)); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "bind"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + if (Socket->Connected) { + CxPlatZeroMemory(&MappedAddress, sizeof(MappedAddress)); + CxPlatConvertToMappedV6(&Socket->RemoteAddress, &MappedAddress); + +#if QUIC_ADDRESS_FAMILY_INET6 != AF_INET6 + if (MappedAddress.Ipv6.sin6_family == QUIC_ADDRESS_FAMILY_INET6) { + MappedAddress.Ipv6.sin6_family = AF_INET6; + } +#endif + // + // Create a temporary UDP socket bound to a wildcard port + // and connect this socket to the remote address. + // By doing this, the OS will select a local address for us. + // + uint16_t LocalPortChosen = 0; + QUIC_ADDR TempLocalAddress = {0}; + AssignedLocalAddressLength = sizeof(TempLocalAddress); + Result = + getsockname( + Socket->AuxSocket, + (struct sockaddr*)&TempLocalAddress, + &AssignedLocalAddressLength); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "getsockname"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + LocalPortChosen = TempLocalAddress.Ipv4.sin_port; + TempUdpSocket = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP); + if (TempUdpSocket == INVALID_SOCKET) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "temp udp socket"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + Option = FALSE; + Result = + setsockopt( + TempUdpSocket, + IPPROTO_IPV6, + IPV6_V6ONLY, + (char*)&Option, + sizeof(Option)); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "Set IPV6_V6ONLY (temp udp socket)"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + CxPlatZeroMemory(&TempLocalAddress, sizeof(TempLocalAddress)); + CxPlatConvertToMappedV6(&Socket->LocalAddress, &TempLocalAddress); + TempLocalAddress.Ipv4.sin_port = 0; + Result = + bind( + TempUdpSocket, + (struct sockaddr*)&TempLocalAddress, + sizeof(TempLocalAddress)); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "bind (temp udp socket)"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + Result = + connect( + TempUdpSocket, + (struct sockaddr*)&MappedAddress, + sizeof(MappedAddress)); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "connect failed (temp udp socket)"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + + AssignedLocalAddressLength = sizeof(Socket->LocalAddress); + Result = + getsockname( + TempUdpSocket, + (struct sockaddr*)&Socket->LocalAddress, + &AssignedLocalAddressLength); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "getsockname (temp udp socket)"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + CxPlatConvertFromMappedV6(&Socket->LocalAddress, &Socket->LocalAddress); + Socket->LocalAddress.Ipv4.sin_port = LocalPortChosen; + CXPLAT_FRE_ASSERT(Socket->LocalAddress.Ipv4.sin_port != 0); + } else { + AssignedLocalAddressLength = sizeof(Socket->LocalAddress); + Result = + getsockname( + Socket->AuxSocket, + (struct sockaddr*)&Socket->LocalAddress, + &AssignedLocalAddressLength); + if (Result == SOCKET_ERROR) { + int WsaError = CxPlatSocketError(); + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + WsaError, + "getsockname"); + CxPlatRwLockReleaseExclusive(&Pool->Lock); + Status = CxPlatQuicErrorFromSocketError(WsaError); + goto Error; + } + CxPlatConvertFromMappedV6(&Socket->LocalAddress, &Socket->LocalAddress); + } + } + + Entry = CxPlatHashtableLookup(&Pool->Sockets, Socket->LocalAddress.Ipv4.sin_port, &Context); + while (Entry != NULL) { + CXPLAT_SOCKET_RAW* Temp = CXPLAT_CONTAINING_RECORD(Entry, CXPLAT_SOCKET_RAW, Entry); + if (CxPlatSocketCompare(Temp, &Socket->LocalAddress, &Socket->RemoteAddress)) { + Status = QUIC_STATUS_ADDRESS_IN_USE; + break; + } + Entry = CxPlatHashtableLookupNext(&Pool->Sockets, &Context); + } + if (QUIC_SUCCEEDED(Status)) { + CxPlatHashtableInsert(&Pool->Sockets, &Socket->Entry, Socket->LocalAddress.Ipv4.sin_port, &Context); + } + + CxPlatRwLockReleaseExclusive(&Pool->Lock); + +Error: + + if (QUIC_FAILED(Status) && Socket->AuxSocket != INVALID_SOCKET) { + CxPlatCloseSocket(Socket->AuxSocket); + } + + if (TempUdpSocket != INVALID_SOCKET) { + CxPlatCloseSocket(TempUdpSocket); + } + + return Status; +} + +#endif // _KERNEL_MODE \ No newline at end of file diff --git a/src/platform/datapath_raw_socket_win.c b/src/platform/datapath_raw_socket_win.c index a6eb168ccb..f12d640f13 100644 --- a/src/platform/datapath_raw_socket_win.c +++ b/src/platform/datapath_raw_socket_win.c @@ -14,8 +14,6 @@ #include "datapath_raw_socket_win.c.clog.h" #endif -#define SocketError() WSAGetLastError() - #pragma warning(disable:4116) // unnamed type definition in parentheses #pragma warning(disable:4100) // unreferenced formal parameter @@ -23,6 +21,31 @@ // Socket Pool Logic // +#ifdef _KERNEL_MODE + +BOOLEAN +CxPlatSockPoolInitialize( + _Inout_ CXPLAT_SOCKET_POOL* Pool + ) +{ + if (!CxPlatHashtableInitializeEx(&Pool->Sockets, CXPLAT_HASH_MIN_SIZE)) { + return FALSE; + } + // WskRegister etc. is called in DataPathInitialize + return TRUE; +} + +void +CxPlatSockPoolUninitialize( + _Inout_ CXPLAT_SOCKET_POOL* Pool + ) +{ + UNREFERENCED_PARAMETER(Pool); + // WskDeregister etc. is called in DataPathUninitialize +} + +#else + BOOLEAN CxPlatSockPoolInitialize( _Inout_ CXPLAT_SOCKET_POOL* Pool @@ -53,6 +76,8 @@ CxPlatSockPoolUninitialize( (void)WSACleanup(); } +#endif // _KERNEL_MODE + _IRQL_requires_max_(PASSIVE_LEVEL) QUIC_STATUS RawResolveRoute( diff --git a/src/platform/datapath_raw_socket_winuser.c b/src/platform/datapath_raw_socket_winuser.c new file mode 100644 index 0000000000..53773a8c1e --- /dev/null +++ b/src/platform/datapath_raw_socket_winuser.c @@ -0,0 +1,222 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + QUIC raw datapath socket and IP framing abstractions + +--*/ + +#include "datapath_raw_win.h" +#ifdef QUIC_CLOG +#include "datapath_raw_socket_winuser.c.clog.h" +#endif + +#define SocketError() WSAGetLastError() + +#pragma warning(disable:4116) // unnamed type definition in parentheses +#pragma warning(disable:4100) // unreferenced formal parameter + +// +// Socket Pool Logic +// + +BOOLEAN +CxPlatSockPoolInitialize( + _Inout_ CXPLAT_SOCKET_POOL* Pool + ) +{ + if (!CxPlatHashtableInitializeEx(&Pool->Sockets, CXPLAT_HASH_MIN_SIZE)) { + return FALSE; + } + int WsaError; + WSADATA WsaData; + if ((WsaError = WSAStartup(MAKEWORD(2, 2), &WsaData)) != 0) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + WsaError, + "WSAStartup"); + CxPlatHashtableUninitialize(&Pool->Sockets); + return FALSE; + } + return TRUE; +} + +void +CxPlatSockPoolUninitialize( + _Inout_ CXPLAT_SOCKET_POOL* Pool + ) +{ + (void)WSACleanup(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +QUIC_STATUS +RawResolveRoute( + _In_ CXPLAT_SOCKET_RAW* Socket, + _Inout_ CXPLAT_ROUTE* Route, + _In_ uint8_t PathId, + _In_ void* Context, + _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback + ) +{ + NETIO_STATUS Status; + MIB_IPFORWARD_ROW2 IpforwardRow = {0}; + CXPLAT_ROUTE_STATE State = Route->State; + QUIC_ADDR LocalAddress = {0}; + + CXPLAT_DBG_ASSERT(!QuicAddrIsWildCard(&Route->RemoteAddress)); + + Route->State = RouteResolving; + + QuicTraceEvent( + DatapathGetRouteStart, + "[data][%p] Querying route, local=%!ADDR!, remote=%!ADDR!", + Socket, + CASTED_CLOG_BYTEARRAY(sizeof(Route->LocalAddress), &Route->LocalAddress), + CASTED_CLOG_BYTEARRAY(sizeof(Route->RemoteAddress), &Route->RemoteAddress)); + + // + // Find the best next hop IP address. + // + Status = + GetBestRoute2( + NULL, // InterfaceLuid + IFI_UNSPECIFIED, // InterfaceIndex + &Route->LocalAddress, // SourceAddress + &Route->RemoteAddress, // DestinationAddress + 0, // AddressSortOptions + &IpforwardRow, + &LocalAddress); // BestSourceAddress + + if (QUIC_FAILED(Status)) { + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Status, + "GetBestRoute2"); + goto Done; + } + + QuicTraceEvent( + DatapathGetRouteComplete, + "[data][%p] Query route result: %!ADDR!", + Socket, + CASTED_CLOG_BYTEARRAY(sizeof(LocalAddress), &LocalAddress)); + + if (State == RouteSuspected && !QuicAddrCompareIp(&LocalAddress, &Route->LocalAddress)) { + // + // We can't handle local address change here easily due to lack of full migration support. + // + Status = QUIC_STATUS_INVALID_STATE; + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + Status, + "GetBestRoute2 returned different local address for the suspected route"); + goto Done; + } else { + LocalAddress.Ipv4.sin_port = Route->LocalAddress.Ipv4.sin_port; // Preserve local port. + Route->LocalAddress = LocalAddress; + } + + // + // Find the interface that matches the route we just looked up. + // + CXPLAT_LIST_ENTRY* Entry = Socket->RawDatapath->Interfaces.Flink; + for (; Entry != &Socket->RawDatapath->Interfaces; Entry = Entry->Flink) { + CXPLAT_INTERFACE* Interface = CONTAINING_RECORD(Entry, CXPLAT_INTERFACE, Link); + if (Interface->IfIndex == IpforwardRow.InterfaceIndex) { + CXPLAT_DBG_ASSERT(sizeof(Interface->PhysicalAddress) == sizeof(Route->LocalLinkLayerAddress)); + CxPlatCopyMemory(&Route->LocalLinkLayerAddress, Interface->PhysicalAddress, sizeof(Route->LocalLinkLayerAddress)); + CxPlatDpRawAssignQueue(Interface, Route); + break; + } + } + + if (Route->Queue == NULL) { + Status = QUIC_STATUS_NOT_FOUND; + QuicTraceEvent( + DatapathError, + "[data][%p] ERROR, %s.", + Socket, + "no matching interface/queue"); + goto Done; + } + + // + // Map the next hop IP address to a link-layer address. + // + MIB_IPNET_ROW2 IpnetRow = {0}; + IpnetRow.InterfaceLuid = IpforwardRow.InterfaceLuid; + if (QuicAddrIsWildCard(&IpforwardRow.NextHop)) { // On-link? + IpnetRow.Address = Route->RemoteAddress; + } else { + IpnetRow.Address = IpforwardRow.NextHop; + } + + // + // Call GetIpNetEntry2 to see if there's already a cached neighbor. + // + Status = GetIpNetEntry2(&IpnetRow); + QuicTraceLogConnInfo( + RouteResolutionStart, + Context, + "Starting to look up neighbor on Path[%hhu] with status %u", + PathId, + Status); + // + // We need to force neighbor solicitation (NS) if any of the following is true: + // 1. No cached neighbor entry for the given destination address. + // 2. The neighbor entry isn't in a usable state. + // 3. When we are re-resolving a suspected route, the neighbor entry is the same as the existing one. + // + // We queue an operation on the route worker for NS because it involves network IO and + // we don't want our connection worker queue blocked. + // + if ((QUIC_FAILED(Status) || IpnetRow.State <= NlnsIncomplete) || + (State == RouteSuspected && + memcmp( + Route->NextHopLinkLayerAddress, + IpnetRow.PhysicalAddress, + sizeof(Route->NextHopLinkLayerAddress)) == 0)) { + CXPLAT_ROUTE_RESOLUTION_WORKER* Worker = Socket->RawDatapath->RouteResolutionWorker; + CXPLAT_ROUTE_RESOLUTION_OPERATION* Operation = CxPlatPoolAlloc(&Worker->OperationPool); + if (Operation == NULL) { + QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "CXPLAT_DATAPATH", + sizeof(CXPLAT_ROUTE_RESOLUTION_OPERATION)); + Status = QUIC_STATUS_PENDING; + goto Done; + } + Operation->IpnetRow = IpnetRow; + Operation->Context = Context; + Operation->Callback = Callback; + Operation->PathId = PathId; + CxPlatDispatchLockAcquire(&Worker->Lock); + CxPlatListInsertTail(&Worker->Operations, &Operation->WorkerLink); + CxPlatDispatchLockRelease(&Worker->Lock); + CxPlatEventSet(Worker->Ready); + Status = QUIC_STATUS_PENDING; + } else { + CxPlatResolveRouteComplete(Context, Route, IpnetRow.PhysicalAddress, PathId); + } + +Done: + if (Status != QUIC_STATUS_PENDING && QUIC_FAILED(Status)) { + Callback(Context, NULL, PathId, FALSE); + } + + if (Status == QUIC_STATUS_PENDING) { + return QUIC_STATUS_PENDING; + } else { + return HRESULT_FROM_WIN32(Status); + } +} diff --git a/src/platform/datapath_raw_win.c b/src/platform/datapath_raw_win.c index d3f01926cd..ea93dc9b04 100644 --- a/src/platform/datapath_raw_win.c +++ b/src/platform/datapath_raw_win.c @@ -195,8 +195,7 @@ CXPLAT_THREAD_CALLBACK(CxPlatRouteResolutionWorkerThread, Context) CXPLAT_ROUTE_RESOLUTION_OPERATION* Operation = CXPLAT_CONTAINING_RECORD( CxPlatListRemoveHead(&Operations), CXPLAT_ROUTE_RESOLUTION_OPERATION, WorkerLink); - NETIO_STATUS Status = - Status = GetIpNetEntry2(&Operation->IpnetRow); + NETIO_STATUS Status = GetIpNetEntry2(&Operation->IpnetRow); if (Status != ERROR_SUCCESS || Operation->IpnetRow.State <= NlnsIncomplete) { Status = ResolveIpNetEntry2(&Operation->IpnetRow, NULL); @@ -242,5 +241,7 @@ CXPLAT_THREAD_CALLBACK(CxPlatRouteResolutionWorkerThread, Context) CXPLAT_FREE(Operation, QUIC_POOL_ROUTE_RESOLUTION_OPER); } +#ifndef _KERNEL_MODE return 0; +#endif } diff --git a/src/platform/datapath_raw_xdp_win.c b/src/platform/datapath_raw_xdp_wincommon.h similarity index 81% rename from src/platform/datapath_raw_xdp_win.c rename to src/platform/datapath_raw_xdp_wincommon.h index e7a745ea58..5e35a12ad4 100644 --- a/src/platform/datapath_raw_xdp_win.c +++ b/src/platform/datapath_raw_xdp_wincommon.h @@ -16,16 +16,11 @@ #include #include "datapath_raw_win.h" #include "datapath_raw_xdp.h" -#include #include #include #include #include -#ifdef QUIC_CLOG -#include "datapath_raw_xdp_win.c.clog.h" -#endif - #define XDP_MAX_SYNC_WAIT_TIMEOUT_MS 1000 // Used for querying XDP RSS capabilities. typedef struct XDP_DATAPATH { @@ -119,9 +114,9 @@ CreateNoOpEthernetPacket( _Inout_ XDP_TX_PACKET* Packet ) { - ETHERNET_HEADER* Ethernet = (ETHERNET_HEADER*)Packet->FrameBuffer; - IPV4_HEADER* IPv4 = (IPV4_HEADER*)(Ethernet + 1); - UDP_HEADER* UDP = (UDP_HEADER*)(IPv4 + 1); + RAW_ETHERNET_HEADER* Ethernet = (RAW_ETHERNET_HEADER*)Packet->FrameBuffer; + RAW_IPV4_HEADER* IPv4 = (RAW_IPV4_HEADER*)(Ethernet + 1); + RAW_UDP_HEADER* UDP = (RAW_UDP_HEADER*)(IPv4 + 1); // Set Ethernet header memset(Ethernet->Destination, 0xFF, sizeof(Ethernet->Destination)); // Broadcast address @@ -131,7 +126,7 @@ CreateNoOpEthernetPacket( // Set IPv4 header IPv4->VersionAndHeaderLength = 0x45; // Version 4, Header length 20 bytes IPv4->TypeOfService = 0; - IPv4->TotalLength = htons(sizeof(IPV4_HEADER) + sizeof(UDP_HEADER)); + IPv4->TotalLength = htons(sizeof(RAW_IPV4_HEADER) + sizeof(RAW_UDP_HEADER)); IPv4->Identification = 0; IPv4->FlagsAndFragmentOffset = 0; IPv4->TimeToLive = 64; @@ -143,13 +138,13 @@ CreateNoOpEthernetPacket( // Set UDP header UDP->SourcePort = htons(12345); UDP->DestinationPort = htons(80); - UDP->Length = htons(sizeof(UDP_HEADER)); + UDP->Length = htons(sizeof(RAW_UDP_HEADER)); UDP->Checksum = 0; // Optional for IPv4 // Calculate IPv4 header checksum uint32_t sum = 0; uint16_t* header = (uint16_t*)IPv4; - for (int i = 0; i < sizeof(IPV4_HEADER) / 2; ++i) { + for (int i = 0; i < sizeof(RAW_IPV4_HEADER) / 2; ++i) { sum += header[i]; } while (sum >> 16) { @@ -158,7 +153,7 @@ CreateNoOpEthernetPacket( IPv4->HeaderChecksum = (uint16_t)~sum; // Set packet length - Packet->Buffer.Length = sizeof(ETHERNET_HEADER) + sizeof(IPV4_HEADER) + sizeof(UDP_HEADER); + Packet->Buffer.Length = sizeof(RAW_ETHERNET_HEADER) + sizeof(RAW_IPV4_HEADER) + sizeof(RAW_UDP_HEADER); } QUIC_STATUS @@ -168,6 +163,7 @@ CxPlatGetRssQueueProcessors( _Out_writes_to_(*Count, *Count) uint32_t* Queues ) { + UNREFERENCED_PARAMETER(Xdp); uint32_t TxRingSize = 1; XDP_TX_PACKET TxPacket = { 0 }; CreateNoOpEthernetPacket(&TxPacket); @@ -201,8 +197,8 @@ CxPlatGetRssQueueProcessors( uint32_t Flags = XSK_BIND_FLAG_TX; Status = XskBind(TxXsk, InterfaceIndex, i, Flags); if (QUIC_FAILED(Status)) { - CloseHandle(TxXsk); - if (Status == E_INVALIDARG) { // No more queues. Break out. + CxPlatCloseHandle(TxXsk); + if (Status == QUIC_STATUS_INVALID_PARAMETER) { // No more queues. Break out. *Count = i; break; // Expected failure if there is no more queue. } @@ -221,7 +217,7 @@ CxPlatGetRssQueueProcessors( "[ lib] ERROR, %u, %s.", Status, "XskActivate (GetRssQueueProcessors)"); - CloseHandle(TxXsk); + CxPlatCloseHandle(TxXsk); return Status; } @@ -249,11 +245,15 @@ CxPlatGetRssQueueProcessors( uint32_t CompIndex; if (XskRingConsumerReserve(&TxCompletionRing, MAXUINT32, &CompIndex) == 0) { - CloseHandle(TxXsk); + CxPlatCloseHandle(TxXsk); return E_ABORT; } XskRingConsumerRelease(&TxCompletionRing, 1); +#ifdef _KERNEL_MODE + // TODO: correct? + Queues[i] = CxPlatProcCurrentNumber(); +#else PROCESSOR_NUMBER ProcNumber; uint32_t ProcNumberSize = sizeof(PROCESSOR_NUMBER); Status = XskGetSockopt(TxXsk, XSK_SOCKOPT_TX_PROCESSOR_AFFINITY, &ProcNumber, &ProcNumberSize); @@ -261,67 +261,14 @@ CxPlatGetRssQueueProcessors( const CXPLAT_PROCESSOR_GROUP_INFO* Group = &CxPlatProcessorGroupInfo[ProcNumber.Group]; Queues[i] = Group->Offset + (ProcNumber.Number % Group->Count); +#endif - CloseHandle(TxXsk); + CxPlatCloseHandle(TxXsk); } return QUIC_STATUS_SUCCESS; } -_IRQL_requires_max_(PASSIVE_LEVEL) -void -CxPlatXdpReadConfig( - _Inout_ XDP_DATAPATH* Xdp - ) -{ - // - // Default config. - // - Xdp->RxBufferCount = 8192; - Xdp->RxRingSize = 256; - Xdp->TxBufferCount = 8192; - Xdp->TxRingSize = 256; - Xdp->TxAlwaysPoke = FALSE; - - // - // Read config from config file. - // - FILE *File = fopen("xdp.ini", "r"); - if (File == NULL) { - return; - } - - char Line[256]; - while (fgets(Line, sizeof(Line), File) != NULL) { - char* Value = strchr(Line, '='); - if (Value == NULL) { - continue; - } - *Value++ = '\0'; - if (Value[strlen(Value) - 1] == '\n') { - Value[strlen(Value) - 1] = '\0'; - } - - if (strcmp(Line, "RxBufferCount") == 0) { - Xdp->RxBufferCount = strtoul(Value, NULL, 10); - } else if (strcmp(Line, "RxRingSize") == 0) { - Xdp->RxRingSize = strtoul(Value, NULL, 10); - } else if (strcmp(Line, "TxBufferCount") == 0) { - Xdp->TxBufferCount = strtoul(Value, NULL, 10); - } else if (strcmp(Line, "TxRingSize") == 0) { - Xdp->TxRingSize = strtoul(Value, NULL, 10); - } else if (strcmp(Line, "TxAlwaysPoke") == 0) { - Xdp->TxAlwaysPoke = !!strtoul(Value, NULL, 10); - } else if (strcmp(Line, "SkipXsum") == 0) { - BOOLEAN State = !!strtoul(Value, NULL, 10); - Xdp->SkipXsum = State; - printf("SkipXsum: %u\n", State); - } - } - - fclose(File); -} - _IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatDpRawInterfaceUninitialize( @@ -335,44 +282,44 @@ CxPlatDpRawInterfaceUninitialize( XDP_QUEUE *Queue = &Interface->Queues[i]; if (Queue->TxXsk != NULL) { - CloseHandle(Queue->TxXsk); + CxPlatCloseHandle(Queue->TxXsk); } if (Queue->TxBuffers != NULL) { - CxPlatFree(Queue->TxBuffers, TX_BUFFER_TAG); + CXPLAT_FREE(Queue->TxBuffers, TX_BUFFER_TAG); } if (Queue->RxProgram != NULL) { - CloseHandle(Queue->RxProgram); + CxPlatCloseHandle(Queue->RxProgram); } if (Queue->RxXsk != NULL) { - CloseHandle(Queue->RxXsk); + CxPlatCloseHandle(Queue->RxXsk); } if (Queue->RxBuffers != NULL) { - CxPlatFree(Queue->RxBuffers, RX_BUFFER_TAG); + CXPLAT_FREE(Queue->RxBuffers, RX_BUFFER_TAG); } CxPlatLockUninitialize(&Queue->TxLock); } if (Interface->Queues != NULL) { - CxPlatFree(Interface->Queues, QUEUE_TAG); + CXPLAT_FREE(Interface->Queues, QUEUE_TAG); } if (Interface->Rules != NULL) { for (uint8_t i = 0; i < Interface->RuleCount; ++i) { if (Interface->Rules[i].Pattern.IpPortSet.PortSet.PortSet) { - CxPlatFree( + CXPLAT_FREE( (uint8_t*)Interface->Rules[i].Pattern.IpPortSet.PortSet.PortSet, PORT_SET_TAG); } } - CxPlatFree(Interface->Rules, RULE_TAG); + CXPLAT_FREE(Interface->Rules, RULE_TAG); } if (Interface->XdpHandle) { - CloseHandle(Interface->XdpHandle); + CxPlatCloseHandle(Interface->XdpHandle); } CxPlatLockUninitialize(&Interface->RuleLock); @@ -458,10 +405,20 @@ CxPlatDpRawInterfaceInitialize( CxPlatZeroMemory(Interface->Queues, Interface->QueueCount * sizeof(*Interface->Queues)); +#ifdef _KERNEL_MODE + FILE_IO_COMPLETION_NOTIFICATION_INFORMATION IoCompletion = {0}; + IoCompletion.Flags = FILE_SKIP_COMPLETION_PORT_ON_SUCCESS | FILE_SKIP_SET_EVENT_ON_HANDLE; +#endif for (uint8_t i = 0; i < Interface->QueueCount; i++) { XDP_QUEUE* Queue = &Interface->Queues[i]; - +#ifdef _KERNEL_MODE +#pragma warning(push) +#pragma warning(disable:6385) +#endif Queue->RssProcessor = (uint16_t)Processors[i]; // TODO - Should memory be aligned with this? +#ifdef _KERNEL_MODE +#pragma warning(pop) +#endif Queue->Interface = Interface; InitializeSListHead(&Queue->RxPool); InitializeSListHead(&Queue->TxPool); @@ -576,6 +533,7 @@ CxPlatDpRawInterfaceInitialize( &Queue->RxPool, (PSLIST_ENTRY)&Queue->RxBuffers[j * RxPacketSize]); } +#ifndef _KERNEL_MODE // // Disable automatic IO completions being queued if the call completes // synchronously. @@ -591,6 +549,29 @@ CxPlatDpRawInterfaceInitialize( "SetFileCompletionNotificationModes"); goto Error; } +#else + { + IO_STATUS_BLOCK IoStatusBlock; + + CxPlatZeroMemory( + &Queue->RxIoSqe.DatapathSqe.Sqe.Overlapped, + sizeof(Queue->RxIoSqe.DatapathSqe.Sqe.Overlapped)); + if (S_OK != + (Status = ZwSetInformationFile( + (HANDLE)Queue->RxXsk, + // (IO_STATUS_BLOCK*)&Queue->RxIoSqe.DatapathSqe.Sqe.Overlapped, + &IoStatusBlock, + &IoCompletion, + sizeof(IoCompletion), + FileIoCompletionNotificationInformation))) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "ZwSetInformationFile"); + } + } +#endif // // TX datapath. @@ -699,6 +680,7 @@ CxPlatDpRawInterfaceInitialize( &Queue->TxPool, (PSLIST_ENTRY)&Queue->TxBuffers[j * sizeof(XDP_TX_PACKET)]); } +#ifndef _KERNEL_MODE // // Disable automatic IO completions being queued if the call completes // synchronously. @@ -714,6 +696,29 @@ CxPlatDpRawInterfaceInitialize( "SetFileCompletionNotificationModes"); goto Error; } +#else + { + IO_STATUS_BLOCK IoStatusBlock; + + CxPlatZeroMemory( + &Queue->TxIoSqe.DatapathSqe.Sqe.Overlapped, + sizeof(Queue->TxIoSqe.DatapathSqe.Sqe.Overlapped)); + if (S_OK != + (Status = ZwSetInformationFile( + (HANDLE)Queue->TxXsk, + &IoStatusBlock, + // (IO_STATUS_BLOCK*)&Queue->TxIoSqe.DatapathSqe.Sqe.Overlapped, + &IoCompletion, + sizeof(IoCompletion), + FileIoCompletionNotificationInformation))) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "ZwSetInformationFile"); + } + } +#endif } // @@ -795,7 +800,7 @@ CxPlatDpRawInterfaceUpdateRules( } if (Queue->RxProgram != NULL) { - CloseHandle(Queue->RxProgram); + CxPlatCloseHandle(Queue->RxProgram); } Queue->RxProgram = NewRxProgram; @@ -847,7 +852,7 @@ CxPlatDpRawInterfaceAddRules( } if (Interface->Rules != NULL) { - CxPlatFree(Interface->Rules, RULE_TAG); + CXPLAT_FREE(Interface->Rules, RULE_TAG); } Interface->Rules = NewRules; @@ -933,254 +938,6 @@ CxPlatDpRawGetDatapathSize( return sizeof(XDP_DATAPATH) + (PartitionCount * sizeof(XDP_PARTITION)); } -_IRQL_requires_max_(PASSIVE_LEVEL) -QUIC_STATUS -CxPlatDpRawInitialize( - _Inout_ CXPLAT_DATAPATH_RAW* Datapath, - _In_ uint32_t ClientRecvContextLength, - _In_ CXPLAT_WORKER_POOL* WorkerPool, - _In_opt_ const QUIC_EXECUTION_CONFIG* Config - ) -{ - XDP_DATAPATH* Xdp = (XDP_DATAPATH*)Datapath; - PMIB_IF_TABLE2 pIfTable = NULL; - QUIC_STATUS Status = QUIC_STATUS_SUCCESS; - - if (WorkerPool == NULL) { - return QUIC_STATUS_INVALID_PARAMETER; - } - - CxPlatListInitializeHead(&Xdp->Interfaces); - - CxPlatXdpReadConfig(Xdp); - Xdp->PollingIdleTimeoutUs = Config ? Config->PollingIdleTimeoutUs : 0; - - if (Config && Config->ProcessorCount) { - Xdp->PartitionCount = Config->ProcessorCount; - for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { - Xdp->Partitions[i].Processor = Config->ProcessorList[i]; - } - } else { - Xdp->PartitionCount = CxPlatProcCount(); - for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { - Xdp->Partitions[i].Processor = (uint16_t)i; - } - } - - if (GetIfTable2(&pIfTable) != NO_ERROR) { - Status = QUIC_STATUS_INTERNAL_ERROR; - goto Error; - } - - PIP_ADAPTER_ADDRESSES Adapters = NULL; - ULONG Error; - ULONG AdaptersBufferSize = 15000; // 15 KB buffer for GAA to start with. - ULONG Iterations = 0; - ULONG flags = // skip info that we don't need. - GAA_FLAG_INCLUDE_PREFIX | - GAA_FLAG_SKIP_UNICAST | - GAA_FLAG_SKIP_ANYCAST | - GAA_FLAG_SKIP_MULTICAST | - GAA_FLAG_SKIP_DNS_SERVER | - GAA_FLAG_SKIP_DNS_INFO; - - do { - Adapters = (IP_ADAPTER_ADDRESSES*)CXPLAT_ALLOC_NONPAGED(AdaptersBufferSize, ADAPTER_TAG); - if (Adapters == NULL) { - QuicTraceEvent( - AllocFailure, - "Allocation of '%s' failed. (%llu bytes)", - "XDP interface", - AdaptersBufferSize); - Status = QUIC_STATUS_OUT_OF_MEMORY; - goto Error; - } - - Error = - GetAdaptersAddresses(AF_UNSPEC, flags, NULL, Adapters, &AdaptersBufferSize); - if (Error == ERROR_BUFFER_OVERFLOW) { - CxPlatFree(Adapters, ADAPTER_TAG); - Adapters = NULL; - } else { - break; - } - - Iterations++; - } while ((Error == ERROR_BUFFER_OVERFLOW) && (Iterations < 3)); // retry up to 3 times. - - if (Error == NO_ERROR) { - for (PIP_ADAPTER_ADDRESSES Adapter = Adapters; Adapter != NULL; Adapter = Adapter->Next) { - if (Adapter->IfType == IF_TYPE_ETHERNET_CSMACD && - Adapter->OperStatus == IfOperStatusUp && - Adapter->PhysicalAddressLength == ETH_MAC_ADDR_LEN) { - XDP_INTERFACE* Interface = CXPLAT_ALLOC_NONPAGED(sizeof(XDP_INTERFACE), IF_TAG); - if (Interface == NULL) { - QuicTraceEvent( - AllocFailure, - "Allocation of '%s' failed. (%llu bytes)", - "XDP interface", - sizeof(*Interface)); - Status = QUIC_STATUS_OUT_OF_MEMORY; - goto Error; - } - CxPlatZeroMemory(Interface, sizeof(*Interface)); - Interface->ActualIfIndex = Interface->IfIndex = Adapter->IfIndex; - memcpy( - Interface->PhysicalAddress, Adapter->PhysicalAddress, - sizeof(Interface->PhysicalAddress)); - - // Look for VF which associated with Adapter - // It has same MAC address. and empirically these flags - /* TODO - Currently causes issues some times - for (int i = 0; i < (int) pIfTable->NumEntries; i++) { - MIB_IF_ROW2* pIfRow = &pIfTable->Table[i]; - if (!pIfRow->InterfaceAndOperStatusFlags.FilterInterface && - pIfRow->InterfaceAndOperStatusFlags.HardwareInterface && - pIfRow->InterfaceAndOperStatusFlags.ConnectorPresent && - pIfRow->PhysicalMediumType == NdisPhysicalMedium802_3 && - memcmp(&pIfRow->PhysicalAddress, &Adapter->PhysicalAddress, - Adapter->PhysicalAddressLength) == 0) { - Interface->ActualIfIndex = pIfRow->InterfaceIndex; - QuicTraceLogInfo( - FoundVF, - "[ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu", - Xdp, - Interface->IfIndex, - Interface->ActualIfIndex); - break; // assuming there is 1:1 matching - } - }*/ - - QuicTraceLogVerbose( - XdpInterfaceInitialize, - "[ixdp][%p] Initializing interface %u", - Interface, - Interface->ActualIfIndex); - - Status = - CxPlatDpRawInterfaceInitialize( - Xdp, Interface, ClientRecvContextLength); - if (Status == QUIC_STATUS_FILE_NOT_FOUND && CxPlatListIsEmpty(&Xdp->Interfaces)) { - // - // FILE_NOT_FOUND for the first interface means that XDP is not available on this system. - // - Status = QUIC_STATUS_NOT_SUPPORTED; - break; - } else if (QUIC_FAILED(Status)) { - QuicTraceEvent( - LibraryErrorStatus, - "[ lib] ERROR, %u, %s.", - Status, - "CxPlatDpRawInterfaceInitialize"); - CxPlatFree(Interface, IF_TAG); - continue; - } - CxPlatListInsertTail(&Xdp->Interfaces, &Interface->Link); - } - } - } else { - Status = HRESULT_FROM_WIN32(Error); - QuicTraceEvent( - LibraryErrorStatus, - "[ lib] ERROR, %u, %s.", - Status, - "CxPlatThreadCreate"); - goto Error; - } - - if (CxPlatListIsEmpty(&Xdp->Interfaces)) { - if (Status == QUIC_STATUS_NOT_SUPPORTED) { - QuicTraceEvent( - LibraryError, - "[ lib] ERROR, %s.", - "XDP is not supported on this system"); - } else { - QuicTraceEvent( - LibraryError, - "[ lib] ERROR, %s.", - "no XDP capable interface"); - Status = QUIC_STATUS_NOT_FOUND; - } - goto Error; - } - - Xdp->Running = TRUE; - CxPlatRefInitialize(&Xdp->RefCount); - for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { - - XDP_PARTITION* Partition = &Xdp->Partitions[i]; - if (Partition->Queues == NULL) { continue; } // No RSS queues for this partition. - - Partition->Xdp = Xdp; - Partition->PartitionIndex = (uint16_t)i; - Partition->Ec.Ready = TRUE; - Partition->Ec.NextTimeUs = UINT64_MAX; - Partition->Ec.Callback = CxPlatXdpExecute; - Partition->Ec.Context = &Xdp->Partitions[i]; - CxPlatSqeInitializeEx(CxPlatIoXdpShutdownEventComplete, &Partition->ShutdownSqe); - CxPlatRefIncrement(&Xdp->RefCount); - Partition->EventQ = CxPlatWorkerPoolGetEventQ(WorkerPool, (uint16_t)i); - - uint32_t QueueCount = 0; - XDP_QUEUE* Queue = Partition->Queues; - while (Queue) { - if (!CxPlatEventQAssociateHandle(Partition->EventQ, Queue->RxXsk)) { - QuicTraceEvent( - LibraryErrorStatus, - "[ lib] ERROR, %u, %s.", - GetLastError(), - "CreateIoCompletionPort(RX)"); - } - if (!CxPlatEventQAssociateHandle(Partition->EventQ, Queue->TxXsk)) { - QuicTraceEvent( - LibraryErrorStatus, - "[ lib] ERROR, %u, %s.", - GetLastError(), - "CreateIoCompletionPort(TX)"); - } - QuicTraceLogVerbose( - XdpQueueStart, - "[ xdp][%p] XDP queue start on partition %p", - Queue, - Partition); - ++QueueCount; - Queue = Queue->Next; - } - - QuicTraceLogVerbose( - XdpWorkerStart, - "[ xdp][%p] XDP partition start, %u queues", - Partition, - QueueCount); - UNREFERENCED_PARAMETER(QueueCount); - - CxPlatAddExecutionContext(WorkerPool, &Partition->Ec, Partition->PartitionIndex); - } - Status = QUIC_STATUS_SUCCESS; - - QuicTraceLogVerbose( - XdpInitialize, - "[ xdp][%p] XDP initialized, %u procs", - Xdp, - Xdp->PartitionCount); - -Error: - if (pIfTable != NULL) { - FreeMibTable(pIfTable); - } - - if (QUIC_FAILED(Status)) { - while (!CxPlatListIsEmpty(&Xdp->Interfaces)) { - XDP_INTERFACE* Interface = - CONTAINING_RECORD(CxPlatListRemoveHead(&Xdp->Interfaces), XDP_INTERFACE, Link); - CxPlatDpRawInterfaceUninitialize(Interface); - CxPlatFree(Interface, IF_TAG); - } - } - - return Status; -} - _IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatDpRawRelease( @@ -1200,7 +957,7 @@ CxPlatDpRawRelease( XDP_INTERFACE* Interface = CONTAINING_RECORD(CxPlatListRemoveHead(&Xdp->Interfaces), XDP_INTERFACE, Link); CxPlatDpRawInterfaceUninitialize(Interface); - CxPlatFree(Interface, IF_TAG); + CXPLAT_FREE(Interface, IF_TAG); } CxPlatDataPathUninitializeComplete((CXPLAT_DATAPATH_RAW*)Xdp); } @@ -1554,7 +1311,7 @@ CxPlatXdpRx( QuicTraceEvent( LibraryErrorStatus, "[ lib] ERROR, %u, %s.", - SUCCEEDED(XskStatus) ? ErrorStatus : XskStatus, + XDP_SUCCEEDED(XskStatus) ? ErrorStatus : XskStatus, "XSK_SOCKOPT_RX_ERROR"); Queue->Error = TRUE; } @@ -1724,7 +1481,7 @@ CxPlatXdpTx( QuicTraceEvent( LibraryErrorStatus, "[ lib] ERROR, %u, %s.", - SUCCEEDED(XskStatus) ? ErrorStatus : XskStatus, + XDP_SUCCEEDED(XskStatus) ? ErrorStatus : XskStatus, "XSK_SOCKOPT_TX_ERROR"); Queue->Error = TRUE; } @@ -1749,11 +1506,11 @@ CxPlatXdpExecute( Partition); XDP_QUEUE* Queue = Partition->Queues; while (Queue) { - CancelIoEx(Queue->RxXsk, NULL); - CloseHandle(Queue->RxXsk); + CxPlatCancelIo(Queue->RxXsk); + CxPlatCloseHandle(Queue->RxXsk); Queue->RxXsk = NULL; - CancelIoEx(Queue->TxXsk, NULL); - CloseHandle(Queue->TxXsk); + CxPlatCancelIo(Queue->TxXsk); + CxPlatCloseHandle(Queue->TxXsk); Queue->TxXsk = NULL; Queue = Queue->Next; } @@ -1794,7 +1551,7 @@ CxPlatXdpExecute( &Queue->RxIoSqe.Overlapped); if (hr == HRESULT_FROM_WIN32(ERROR_IO_PENDING)) { Queue->RxQueued = TRUE; - } else if (hr == S_OK) { + } else if (hr == QUIC_STATUS_SUCCESS) { Partition->Ec.Ready = TRUE; } else { QuicTraceEvent( @@ -1818,7 +1575,7 @@ CxPlatXdpExecute( &Queue->TxIoSqe.Overlapped); if (hr == HRESULT_FROM_WIN32(ERROR_IO_PENDING)) { Queue->TxQueued = TRUE; - } else if (hr == S_OK) { + } else if (hr == QUIC_STATUS_SUCCESS) { Partition->Ec.Ready = TRUE; } else { QuicTraceEvent( diff --git a/src/platform/datapath_raw_xdp_winkernel.c b/src/platform/datapath_raw_xdp_winkernel.c new file mode 100644 index 0000000000..888749706e --- /dev/null +++ b/src/platform/datapath_raw_xdp_winkernel.c @@ -0,0 +1,197 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + QUIC XDP Datapath Implementation (User Mode) + +--*/ + +#define _CRT_SECURE_NO_WARNINGS 1 // TODO - Remove + +#include "datapath_raw_xdp_wincommon.h" + +#ifdef QUIC_CLOG +#include "datapath_raw_xdp_winkernel.c.clog.h" +#endif + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +CxPlatXdpReadConfig( + _Inout_ XDP_DATAPATH* Xdp + ) +{ + // + // Default config. + // + Xdp->RxBufferCount = 8192; + Xdp->RxRingSize = 256; + Xdp->TxBufferCount = 8192; + Xdp->TxRingSize = 256; + Xdp->TxAlwaysPoke = FALSE; + + // TODO: implement config reader + +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +QUIC_STATUS +CxPlatDpRawInitialize( + _Inout_ CXPLAT_DATAPATH_RAW* Datapath, + _In_ uint32_t ClientRecvContextLength, + _In_ CXPLAT_WORKER_POOL* WorkerPool, + _In_opt_ const QUIC_EXECUTION_CONFIG* Config + ) +{ + XDP_DATAPATH* Xdp = (XDP_DATAPATH*)Datapath; + QUIC_STATUS Status = QUIC_STATUS_SUCCESS; + PMIB_IF_TABLE2 pIfTable = NULL; + + if (WorkerPool == NULL) { + return QUIC_STATUS_INVALID_PARAMETER; + } + + CxPlatListInitializeHead(&Xdp->Interfaces); + + CxPlatXdpReadConfig(Xdp); + Xdp->PollingIdleTimeoutUs = Config ? Config->PollingIdleTimeoutUs : 0; + + if (Config && Config->ProcessorCount) { + Xdp->PartitionCount = Config->ProcessorCount; + for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { + Xdp->Partitions[i].Processor = Config->ProcessorList[i]; + } + } else { + Xdp->PartitionCount = CxPlatProcCount(); + for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { + Xdp->Partitions[i].Processor = (uint16_t)i; + } + } + + if (QUIC_FAILED(GetIfTable2(&pIfTable))) { + Status = QUIC_STATUS_INTERNAL_ERROR; + goto Exit; + } + + for (ULONG i = 0; i < pIfTable->NumEntries; i++) { + MIB_IF_ROW2* pIfRow = &pIfTable->Table[i]; + + if (pIfRow->Type == IF_TYPE_ETHERNET_CSMACD && + pIfRow->OperStatus == IfOperStatusUp && + pIfRow->PhysicalAddressLength == ETH_MAC_ADDR_LEN && + pIfRow->InterfaceAndOperStatusFlags.ConnectorPresent) { + XDP_INTERFACE* Interface = CXPLAT_ALLOC_NONPAGED(sizeof(XDP_INTERFACE), IF_TAG); + if (Interface == NULL) { + QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + sizeof(*Interface)); + Status = QUIC_STATUS_OUT_OF_MEMORY; + goto Exit; + } + RtlZeroMemory(Interface, sizeof(XDP_INTERFACE)); + Interface->ActualIfIndex = Interface->IfIndex = pIfRow->InterfaceIndex; + RtlCopyMemory( + Interface->PhysicalAddress, + pIfRow->PhysicalAddress, + min(pIfRow->PhysicalAddressLength, sizeof(Interface->PhysicalAddress)) + ); + + QuicTraceLogVerbose( + XdpInterfaceInitialize, + "[ixdp][%p] Initializing interface %u", + Interface, + Interface->ActualIfIndex); + + Status = + CxPlatDpRawInterfaceInitialize( + Xdp, Interface, ClientRecvContextLength); + if (QUIC_FAILED(Status)) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatDpRawInterfaceInitialize"); + CXPLAT_FREE(Interface, IF_TAG); + continue; + } + + CxPlatListInsertTail(&Xdp->Interfaces, &Interface->Link); + } + } + + if (CxPlatListIsEmpty(&Xdp->Interfaces)) { + QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); + Status = QUIC_STATUS_NOT_FOUND; + goto Exit; + } + + FILE_IO_COMPLETION_NOTIFICATION_INFORMATION IoCompletion = {0}; + IoCompletion.Flags = FILE_SKIP_COMPLETION_PORT_ON_SUCCESS | FILE_SKIP_SET_EVENT_ON_HANDLE; + Xdp->Running = TRUE; + CxPlatRefInitialize(&Xdp->RefCount); + for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { + + XDP_PARTITION* Partition = &Xdp->Partitions[i]; + if (Partition->Queues == NULL) { continue; } // No RSS queues for this partition. + + Partition->Xdp = Xdp; + Partition->PartitionIndex = (uint16_t)i; + Partition->Ec.Ready = TRUE; + Partition->Ec.NextTimeUs = UINT64_MAX; + Partition->Ec.Callback = CxPlatXdpExecute; + Partition->Ec.Context = &Xdp->Partitions[i]; + Partition->ShutdownSqe.CqeType = CXPLAT_CQE_TYPE_SOCKET_SHUTDOWN; + CxPlatRefIncrement(&Xdp->RefCount); + Partition->EventQ = CxPlatWorkerPoolGetEventQ(WorkerPool, (uint16_t)i); + + uint32_t QueueCount = 0; + XDP_QUEUE* Queue = Partition->Queues; + while (Queue) { + QuicTraceLogVerbose( + XdpQueueStart, + "[ xdp][%p] XDP queue start on partition %p", + Queue, + Partition); + ++QueueCount; + Queue = Queue->Next; + } + + QuicTraceLogVerbose( + XdpWorkerStart, + "[ xdp][%p] XDP partition start, %u queues", + Partition, + QueueCount); + UNREFERENCED_PARAMETER(QueueCount); + + CxPlatAddExecutionContext(WorkerPool, &Partition->Ec, Partition->PartitionIndex); + } + Status = QUIC_STATUS_SUCCESS; + + QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); + +Exit: + if (pIfTable != NULL) { + FreeMibTable(pIfTable); + } + + if (!NT_SUCCESS(Status)) { + while (!CxPlatListIsEmpty(&Xdp->Interfaces)) { + XDP_INTERFACE* Interface = CONTAINING_RECORD(CxPlatListRemoveHead(&Xdp->Interfaces), XDP_INTERFACE, Link); + CxPlatDpRawInterfaceUninitialize(Interface); + CXPLAT_FREE(Interface, IF_TAG); + } + } + + return Status; +} diff --git a/src/platform/datapath_raw_xdp_winuser.c b/src/platform/datapath_raw_xdp_winuser.c new file mode 100644 index 0000000000..d8d1d7983f --- /dev/null +++ b/src/platform/datapath_raw_xdp_winuser.c @@ -0,0 +1,315 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + QUIC XDP Datapath Implementation (User Mode) + +--*/ + +#define _CRT_SECURE_NO_WARNINGS 1 // TODO - Remove + +#include "datapath_raw_xdp_wincommon.h" +#include + +#ifdef QUIC_CLOG +#include "datapath_raw_xdp_winuser.c.clog.h" +#endif + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +CxPlatXdpExecute( + _Inout_ void* Context, + _Inout_ CXPLAT_EXECUTION_STATE* State + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +CxPlatXdpReadConfig( + _Inout_ XDP_DATAPATH* Xdp + ) +{ + // + // Default config. + // + Xdp->RxBufferCount = 8192; + Xdp->RxRingSize = 256; + Xdp->TxBufferCount = 8192; + Xdp->TxRingSize = 256; + Xdp->TxAlwaysPoke = FALSE; + + // + // Read config from config file. + // + FILE *File = fopen("xdp.ini", "r"); + if (File == NULL) { + return; + } + + char Line[256]; + while (fgets(Line, sizeof(Line), File) != NULL) { + char* Value = strchr(Line, '='); + if (Value == NULL) { + continue; + } + *Value++ = '\0'; + if (Value[strlen(Value) - 1] == '\n') { + Value[strlen(Value) - 1] = '\0'; + } + + if (strcmp(Line, "RxBufferCount") == 0) { + Xdp->RxBufferCount = strtoul(Value, NULL, 10); + } else if (strcmp(Line, "RxRingSize") == 0) { + Xdp->RxRingSize = strtoul(Value, NULL, 10); + } else if (strcmp(Line, "TxBufferCount") == 0) { + Xdp->TxBufferCount = strtoul(Value, NULL, 10); + } else if (strcmp(Line, "TxRingSize") == 0) { + Xdp->TxRingSize = strtoul(Value, NULL, 10); + } else if (strcmp(Line, "TxAlwaysPoke") == 0) { + Xdp->TxAlwaysPoke = !!strtoul(Value, NULL, 10); + } else if (strcmp(Line, "SkipXsum") == 0) { + BOOLEAN State = !!strtoul(Value, NULL, 10); + Xdp->SkipXsum = State; + printf("SkipXsum: %u\n", State); + } + } + + fclose(File); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +QUIC_STATUS +CxPlatDpRawInitialize( + _Inout_ CXPLAT_DATAPATH_RAW* Datapath, + _In_ uint32_t ClientRecvContextLength, + _In_ CXPLAT_WORKER_POOL* WorkerPool, + _In_opt_ const QUIC_EXECUTION_CONFIG* Config + ) +{ + XDP_DATAPATH* Xdp = (XDP_DATAPATH*)Datapath; + QUIC_STATUS Status = QUIC_STATUS_SUCCESS; + PMIB_IF_TABLE2 pIfTable = NULL; + + if (WorkerPool == NULL) { + return QUIC_STATUS_INVALID_PARAMETER; + } + + CxPlatListInitializeHead(&Xdp->Interfaces); + + CxPlatXdpReadConfig(Xdp); + Xdp->PollingIdleTimeoutUs = Config ? Config->PollingIdleTimeoutUs : 0; + + if (Config && Config->ProcessorCount) { + Xdp->PartitionCount = Config->ProcessorCount; + for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { + Xdp->Partitions[i].Processor = Config->ProcessorList[i]; + } + } else { + Xdp->PartitionCount = CxPlatProcCount(); + for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { + Xdp->Partitions[i].Processor = (uint16_t)i; + } + } + + if (GetIfTable2(&pIfTable) != NO_ERROR) { + Status = QUIC_STATUS_INTERNAL_ERROR; + goto Error; + } + + PIP_ADAPTER_ADDRESSES Adapters = NULL; + ULONG Error; + ULONG AdaptersBufferSize = 15000; // 15 KB buffer for GAA to start with. + ULONG Iterations = 0; + ULONG flags = // skip info that we don't need. + GAA_FLAG_INCLUDE_PREFIX | + GAA_FLAG_SKIP_UNICAST | + GAA_FLAG_SKIP_ANYCAST | + GAA_FLAG_SKIP_MULTICAST | + GAA_FLAG_SKIP_DNS_SERVER | + GAA_FLAG_SKIP_DNS_INFO; + + do { + Adapters = (IP_ADAPTER_ADDRESSES*)CXPLAT_ALLOC_NONPAGED(AdaptersBufferSize, ADAPTER_TAG); + if (Adapters == NULL) { + QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + AdaptersBufferSize); + Status = QUIC_STATUS_OUT_OF_MEMORY; + goto Error; + } + + Error = + GetAdaptersAddresses(AF_UNSPEC, flags, NULL, Adapters, &AdaptersBufferSize); + if (Error == ERROR_BUFFER_OVERFLOW) { + CXPLAT_FREE(Adapters, ADAPTER_TAG); + Adapters = NULL; + } else { + break; + } + + Iterations++; + } while ((Error == ERROR_BUFFER_OVERFLOW) && (Iterations < 3)); // retry up to 3 times. + + if (Error == NO_ERROR) { + for (PIP_ADAPTER_ADDRESSES Adapter = Adapters; Adapter != NULL; Adapter = Adapter->Next) { + if (Adapter->IfType == IF_TYPE_ETHERNET_CSMACD && + Adapter->OperStatus == IfOperStatusUp && + Adapter->PhysicalAddressLength == ETH_MAC_ADDR_LEN) { + XDP_INTERFACE* Interface = CXPLAT_ALLOC_NONPAGED(sizeof(XDP_INTERFACE), IF_TAG); + if (Interface == NULL) { + QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "XDP interface", + sizeof(*Interface)); + Status = QUIC_STATUS_OUT_OF_MEMORY; + goto Error; + } + CxPlatZeroMemory(Interface, sizeof(*Interface)); + Interface->ActualIfIndex = Interface->IfIndex = Adapter->IfIndex; + memcpy( + Interface->PhysicalAddress, Adapter->PhysicalAddress, + sizeof(Interface->PhysicalAddress)); + + // Look for VF which associated with Adapter + // It has same MAC address. and empirically these flags + /* TODO - Currently causes issues some times + for (int i = 0; i < (int) pIfTable->NumEntries; i++) { + MIB_IF_ROW2* pIfRow = &pIfTable->Table[i]; + if (!pIfRow->InterfaceAndOperStatusFlags.FilterInterface && + pIfRow->InterfaceAndOperStatusFlags.HardwareInterface && + pIfRow->InterfaceAndOperStatusFlags.ConnectorPresent && + pIfRow->PhysicalMediumType == NdisPhysicalMedium802_3 && + memcmp(&pIfRow->PhysicalAddress, &Adapter->PhysicalAddress, + Adapter->PhysicalAddressLength) == 0) { + Interface->ActualIfIndex = pIfRow->InterfaceIndex; + QuicTraceLogInfo( + FoundVF, + "[ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu", + Xdp, + Interface->IfIndex, + Interface->ActualIfIndex); + break; // assuming there is 1:1 matching + } + }*/ + + QuicTraceLogVerbose( + XdpInterfaceInitialize, + "[ixdp][%p] Initializing interface %u", + Interface, + Interface->ActualIfIndex); + + Status = + CxPlatDpRawInterfaceInitialize( + Xdp, Interface, ClientRecvContextLength); + if (QUIC_FAILED(Status)) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatDpRawInterfaceInitialize"); + CXPLAT_FREE(Interface, IF_TAG); + continue; + } + CxPlatListInsertTail(&Xdp->Interfaces, &Interface->Link); + } + } + } else { + Status = HRESULT_FROM_WIN32(Error); + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + Status, + "CxPlatThreadCreate"); + goto Error; + } + + if (CxPlatListIsEmpty(&Xdp->Interfaces)) { + QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); + Status = QUIC_STATUS_NOT_FOUND; + goto Error; + } + + Xdp->Running = TRUE; + CxPlatRefInitialize(&Xdp->RefCount); + for (uint32_t i = 0; i < Xdp->PartitionCount; i++) { + + XDP_PARTITION* Partition = &Xdp->Partitions[i]; + if (Partition->Queues == NULL) { continue; } // No RSS queues for this partition. + + Partition->Xdp = Xdp; + Partition->PartitionIndex = (uint16_t)i; + Partition->Ec.Ready = TRUE; + Partition->Ec.NextTimeUs = UINT64_MAX; + Partition->Ec.Callback = CxPlatXdpExecute; + Partition->Ec.Context = &Xdp->Partitions[i]; + Partition->ShutdownSqe.CqeType = CXPLAT_CQE_TYPE_SOCKET_SHUTDOWN; + CxPlatRefIncrement(&Xdp->RefCount); + Partition->EventQ = CxPlatWorkerPoolGetEventQ(WorkerPool, (uint16_t)i); + + uint32_t QueueCount = 0; + XDP_QUEUE* Queue = Partition->Queues; + while (Queue) { + if (!CxPlatEventQAssociateHandle(Partition->EventQ, Queue->RxXsk)) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + GetLastError(), + "CreateIoCompletionPort(RX)"); + } + if (!CxPlatEventQAssociateHandle(Partition->EventQ, Queue->TxXsk)) { + QuicTraceEvent( + LibraryErrorStatus, + "[ lib] ERROR, %u, %s.", + GetLastError(), + "CreateIoCompletionPort(TX)"); + } + QuicTraceLogVerbose( + XdpQueueStart, + "[ xdp][%p] XDP queue start on partition %p", + Queue, + Partition); + ++QueueCount; + Queue = Queue->Next; + } + + QuicTraceLogVerbose( + XdpWorkerStart, + "[ xdp][%p] XDP partition start, %u queues", + Partition, + QueueCount); + UNREFERENCED_PARAMETER(QueueCount); + + CxPlatAddExecutionContext(WorkerPool, &Partition->Ec, Partition->PartitionIndex); + } + Status = QUIC_STATUS_SUCCESS; + + QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); + +Error: + if (pIfTable != NULL) { + FreeMibTable(pIfTable); + } + + if (QUIC_FAILED(Status)) { + while (!CxPlatListIsEmpty(&Xdp->Interfaces)) { + XDP_INTERFACE* Interface = + CONTAINING_RECORD(CxPlatListRemoveHead(&Xdp->Interfaces), XDP_INTERFACE, Link); + CxPlatDpRawInterfaceUninitialize(Interface); + CXPLAT_FREE(Interface, IF_TAG); + } + } + + return Status; +} diff --git a/src/platform/datapath_winkernel.c b/src/platform/datapath_winkernel.c index cb7d823256..bde81b968a 100644 --- a/src/platform/datapath_winkernel.c +++ b/src/platform/datapath_winkernel.c @@ -688,7 +688,6 @@ DataPathInitialize( _Out_ CXPLAT_DATAPATH* *NewDataPath ) { - UNREFERENCED_PARAMETER(WorkerPool); QUIC_STATUS Status; WSK_CLIENT_NPI WskClientNpi = { NULL, &WskAppDispatch }; uint32_t DatapathLength; @@ -716,6 +715,19 @@ DataPathInitialize( goto Exit; } } + if (WorkerPool == NULL) { + return QUIC_STATUS_INVALID_PARAMETER; + } + + if (!CxPlatWorkerPoolLazyStart(WorkerPool, Config)) { + Status = QUIC_STATUS_OUT_OF_MEMORY; + goto Exit; + } + + if (!CxPlatWorkerPoolLazyStart(WorkerPool, Config)) { + Status = QUIC_STATUS_OUT_OF_MEMORY; + goto Exit; + } DatapathLength = sizeof(CXPLAT_DATAPATH) + @@ -736,9 +748,11 @@ DataPathInitialize( if (UdpCallbacks) { Datapath->UdpHandlers = *UdpCallbacks; } + Datapath->WorkerPool = WorkerPool; Datapath->ClientRecvDataLength = ClientRecvDataLength; Datapath->ProcCount = (uint32_t)CxPlatProcCount(); Datapath->WskDispatch.WskReceiveFromEvent = CxPlatDataPathSocketReceive; + Datapath->UseTcp = FALSE; Datapath->DatagramStride = ALIGN_UP( sizeof(DATAPATH_RX_PACKET) + @@ -1279,8 +1293,8 @@ SocketCreateUdp( ) { QUIC_STATUS Status = STATUS_SUCCESS; - size_t BindingSize; - CXPLAT_SOCKET* Binding = NULL; + size_t RawBindingSize; + CXPLAT_SOCKET_RAW* RawBinding = NULL; uint32_t Option; if (Datapath == NULL || NewBinding == NULL) { @@ -1288,20 +1302,21 @@ SocketCreateUdp( goto Error; } - BindingSize = - sizeof(CXPLAT_SOCKET) + + RawBindingSize = + CxPlatGetRawSocketSize() + CxPlatProcCount() * sizeof(CXPLAT_RUNDOWN_REF); - Binding = (CXPLAT_SOCKET*)CXPLAT_ALLOC_NONPAGED(BindingSize, QUIC_POOL_SOCKET); - if (Binding == NULL) { + RawBinding = (CXPLAT_SOCKET_RAW*)CXPLAT_ALLOC_NONPAGED(RawBindingSize, QUIC_POOL_SOCKET); + if (RawBinding == NULL) { QuicTraceEvent( AllocFailure, "Allocation of '%s' failed. (%llu bytes)", "CXPLAT_SOCKET", - BindingSize); + RawBindingSize); Status = QUIC_STATUS_OUT_OF_MEMORY; goto Error; } + CXPLAT_SOCKET* Binding = CxPlatRawToSocket(RawBinding); // // Must set output pointer first thing, as the receive path will try to @@ -1316,10 +1331,11 @@ SocketCreateUdp( CASTED_CLOG_BYTEARRAY(Config->LocalAddress ? sizeof(*Config->LocalAddress) : 0, Config->LocalAddress), CASTED_CLOG_BYTEARRAY(Config->RemoteAddress ? sizeof(*Config->RemoteAddress) : 0, Config->RemoteAddress)); - RtlZeroMemory(Binding, BindingSize); + RtlZeroMemory(RawBinding, RawBindingSize); Binding->Datapath = Datapath; Binding->ClientContext = Config->CallbackContext; Binding->Connected = (Config->RemoteAddress != NULL); + Binding->UseTcp = FALSE; if (Config->LocalAddress != NULL) { CxPlatConvertToMappedV6(Config->LocalAddress, &Binding->LocalAddress); } else { @@ -1783,11 +1799,14 @@ SocketCreateUdp( Binding->RemoteAddress.Ipv4.sin_port = 0; } + RawBinding = NULL; + Binding = NULL; + Error: if (QUIC_FAILED(Status)) { - if (Binding != NULL) { - CxPlatSocketDelete(Binding); + if (RawBinding != NULL) { + SocketDelete(CxPlatRawToSocket(RawBinding)); } } @@ -1838,7 +1857,7 @@ CxPlatSocketDeleteComplete( for (uint32_t i = 0; i < CxPlatProcCount(); ++i) { CxPlatRundownUninitialize(&Binding->Rundown[i]); } - CXPLAT_FREE(Binding, QUIC_POOL_SOCKET); + CXPLAT_FREE(CxPlatSocketToRaw(Binding), QUIC_POOL_SOCKET); } IO_COMPLETION_ROUTINE CxPlatDataPathCloseSocketIoCompletion; diff --git a/src/platform/platform.kernel.vcxproj b/src/platform/platform.kernel.vcxproj index 6e1f4506de..9b1ad4b19d 100644 --- a/src/platform/platform.kernel.vcxproj +++ b/src/platform/platform.kernel.vcxproj @@ -23,11 +23,17 @@ - + + + + + + + @@ -85,7 +91,16 @@ - ..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir); + %(AdditionalIncludeDirectories) + + + %(PreprocessorDefinitions) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/platform/platform_internal.h b/src/platform/platform_internal.h index 78e2350c04..a43a83f8b6 100644 --- a/src/platform/platform_internal.h +++ b/src/platform/platform_internal.h @@ -131,6 +131,8 @@ typedef enum CXPLAT_SOCKET_TYPE { #ifdef _KERNEL_MODE +#include + #define CXPLAT_BASE_REG_PATH L"\\Registry\\Machine\\System\\CurrentControlSet\\Services\\MsQuic\\Parameters\\" #define SOCKET PWSK_SOCKET @@ -183,6 +185,9 @@ typedef struct CXPLAT_SOCKET { // BOOLEAN PcpBinding : 1; + BOOLEAN UseTcp : 1; // Not supported. always FALSE + BOOLEAN RawSocketAvailable : 1; + // // UDP socket used for sending/receiving datagrams. // @@ -204,9 +209,6 @@ typedef struct CXPLAT_SOCKET { UCHAR IrpBuffer[sizeof(IRP) + sizeof(IO_STACK_LOCATION)]; }; - uint8_t UseTcp : 1; // always false? - uint8_t RawSocketAvailable : 1; - CXPLAT_RUNDOWN_REF Rundown[0]; // Per-proc } CXPLAT_SOCKET; @@ -287,10 +289,22 @@ typedef struct CXPLAT_DATAPATH { } CXPLAT_DATAPATH; +#ifndef htons +#define htons _byteswap_ushort +#endif + +#ifndef ntohs +#define ntohs _byteswap_ushort +#endif + #ifndef htonl #define htonl _byteswap_ulong #endif +#ifndef ntohl +#define ntohl _byteswap_ulong +#endif + #elif _WIN32 #pragma warning(push) @@ -764,6 +778,7 @@ CxPlatCryptUninitialize( // Platform Worker APIs // +_IRQL_requires_max_(PASSIVE_LEVEL) BOOLEAN CxPlatWorkerPoolLazyStart( _In_ CXPLAT_WORKER_POOL* WorkerPool, @@ -1035,7 +1050,7 @@ typedef struct CXPLAT_DATAPATH { #endif // CX_PLATFORM_LINUX -#if defined(CX_PLATFORM_LINUX) || _WIN32 +#if defined(CX_PLATFORM_LINUX) || _WIN32 || defined(_KERNEL_MODE) typedef struct CXPLAT_SOCKET_RAW CXPLAT_SOCKET_RAW; diff --git a/src/platform/platform_winkernel.c b/src/platform/platform_winkernel.c index 929e9ab149..15a4332a73 100644 --- a/src/platform/platform_winkernel.c +++ b/src/platform/platform_winkernel.c @@ -18,6 +18,9 @@ #include "platform_winkernel.c.clog.h" #endif +CXPLAT_PROCESSOR_INFO* CxPlatProcessorInfo; +CXPLAT_PROCESSOR_GROUP_INFO* CxPlatProcessorGroupInfo; + typedef enum _SYSTEM_INFORMATION_CLASS { SystemBasicInformation = 0 } SYSTEM_INFORMATION_CLASS; @@ -247,6 +250,18 @@ CxPlatRandom( 0); } +_IRQL_requires_max_(DISPATCH_LEVEL) +void +CxPlatDatapathSqeInitialize( + _Out_ DATAPATH_SQE* DatapathSqe, + _In_ uint32_t CqeType + ) +{ + RtlZeroMemory(DatapathSqe, sizeof(*DatapathSqe)); + DatapathSqe->CqeType = CqeType; + DatapathSqe->Sqe.UserData = DatapathSqe; +} + #ifdef DEBUG void diff --git a/src/platform/platform_worker.c b/src/platform/platform_worker.c index 78a139c098..2a5fb4d68a 100644 --- a/src/platform/platform_worker.c +++ b/src/platform/platform_worker.c @@ -121,6 +121,7 @@ WakeCompletion( UNREFERENCED_PARAMETER(Cqe); } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatUpdateExecutionContexts( _In_ CXPLAT_WORKER* Worker @@ -136,6 +137,7 @@ UpdatePollCompletion( CxPlatUpdateExecutionContexts(Worker); } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatWorkerPoolInit( _In_ CXPLAT_WORKER_POOL* WorkerPool @@ -149,6 +151,7 @@ CxPlatWorkerPoolInit( #pragma warning(push) #pragma warning(disable:6385) #pragma warning(disable:6386) // SAL is confused about the worker size +_IRQL_requires_max_(PASSIVE_LEVEL) BOOLEAN CxPlatWorkerPoolLazyStart( _In_ CXPLAT_WORKER_POOL* WorkerPool, @@ -173,7 +176,7 @@ CxPlatWorkerPoolLazyStart( CXPLAT_DBG_ASSERT(WorkerPool->WorkerCount > 0 && WorkerPool->WorkerCount <= UINT16_MAX); const size_t WorkersSize = sizeof(CXPLAT_WORKER) * WorkerPool->WorkerCount; - WorkerPool->Workers = (CXPLAT_WORKER*)CXPLAT_ALLOC_PAGED(WorkersSize, QUIC_POOL_PLATFORM_WORKER); + WorkerPool->Workers = (CXPLAT_WORKER*)CXPLAT_ALLOC_NONPAGED(WorkersSize, QUIC_POOL_PLATFORM_WORKER); if (WorkerPool->Workers == NULL) { QuicTraceEvent( AllocFailure, @@ -302,6 +305,7 @@ CxPlatWorkerPoolLazyStart( } #pragma warning(pop) +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatWorkerPoolUninit( _In_ CXPLAT_WORKER_POOL* WorkerPool @@ -339,6 +343,8 @@ CxPlatWorkerPoolUninit( CxPlatLockUninitialize(&WorkerPool->WorkerLock); } +#ifndef _KERNEL_MODE // Not supported on kernel mode + #define DYNAMIC_POOL_PROCESSING_PERIOD 1000000 // 1 second #define DYNAMIC_POOL_PRUNE_COUNT 8 @@ -401,6 +407,8 @@ CxPlatProcessDynamicPoolAllocators( CxPlatLockRelease(&Worker->ECLock); } +#endif // _KERNEL_MODE + CXPLAT_EVENTQ* CxPlatWorkerPoolGetEventQ( _In_ const CXPLAT_WORKER_POOL* WorkerPool, @@ -412,6 +420,7 @@ CxPlatWorkerPoolGetEventQ( return &WorkerPool->Workers[Index].EventQ; } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatAddExecutionContext( _In_ CXPLAT_WORKER_POOL* WorkerPool, @@ -435,6 +444,7 @@ CxPlatAddExecutionContext( } } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatWakeExecutionContext( _In_ CXPLAT_EXECUTION_CONTEXT* Context @@ -446,6 +456,7 @@ CxPlatWakeExecutionContext( } } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatUpdateExecutionContexts( _In_ CXPLAT_WORKER* Worker @@ -467,6 +478,7 @@ CxPlatUpdateExecutionContexts( } } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatRunExecutionContexts( _In_ CXPLAT_WORKER* Worker, @@ -524,13 +536,14 @@ CxPlatRunExecutionContexts( } } +_IRQL_requires_max_(PASSIVE_LEVEL) void CxPlatProcessEvents( _In_ CXPLAT_WORKER* Worker, _Inout_ CXPLAT_EXECUTION_STATE* State ) { - CXPLAT_CQE Cqes[16]; + CXPLAT_CQE Cqes[CXPLAT_EVENTQ_DEQUEUE_MAX]; uint32_t CqeCount = CxPlatEventQDequeue(&Worker->EventQ, Cqes, ARRAYSIZE(Cqes), State->WaitTime); InterlockedFetchAndSetBoolean(&Worker->Running); if (CqeCount != 0) { @@ -591,10 +604,12 @@ CXPLAT_THREAD_CALLBACK(CxPlatWorkerThread, Context) State.NoWorkCount = 0; } +#ifndef _KERNEL_MODE // Unnecessary on kernel mode if (State.TimeNow - State.LastPoolProcessTime > DYNAMIC_POOL_PROCESSING_PERIOD) { CxPlatProcessDynamicPoolAllocators(Worker); State.LastPoolProcessTime = State.TimeNow; } +#endif // _KERNEL_MODE } Worker->Running = FALSE; diff --git a/src/test/bin/winkernel/msquictest.kernel.vcxproj b/src/test/bin/winkernel/msquictest.kernel.vcxproj index b377bb56b4..b02c3a3a12 100644 --- a/src/test/bin/winkernel/msquictest.kernel.vcxproj +++ b/src/test/bin/winkernel/msquictest.kernel.vcxproj @@ -80,7 +80,14 @@ - ..\..;..\..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\..; + ..\..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir); + %(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/test/bin/winkernel/msquictestpriv.kernel.vcxproj b/src/test/bin/winkernel/msquictestpriv.kernel.vcxproj index e22f2e56b7..3b6a3ed0d8 100644 --- a/src/test/bin/winkernel/msquictestpriv.kernel.vcxproj +++ b/src/test/bin/winkernel/msquictestpriv.kernel.vcxproj @@ -80,7 +80,14 @@ - ..\..;..\..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\..; + ..\..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc; + $(IntDir); + %(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256 diff --git a/src/test/lib/testlib.kernel.vcxproj b/src/test/lib/testlib.kernel.vcxproj index f950bc082d..e02f6bf315 100644 --- a/src/test/lib/testlib.kernel.vcxproj +++ b/src/test/lib/testlib.kernel.vcxproj @@ -81,7 +81,13 @@ - ..\;..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories) + + ..\; + ..\..\inc; + $(SolutionDir)submodules\xdp-for-windows\published\external; + $(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir); + %(AdditionalIncludeDirectories) + Speed true /Gw /kernel /ZH:SHA_256