diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index b8782a806f..8729bac272 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -283,6 +283,71 @@ function Start-RemoteServer { throw "Server failed to start!" } +# Passively starts the server on the remote machine by queuing up a new script to execute. +function Start-RemoteServerPassive { + param ($Session, $Command, $RemoteStateDir, $RunId, $SyncerSecret) + if ($Session -eq "NOT_SUPPORTED") { + Write-Host "Command to start server: $Command" + $headers = @{ + "secret" = "$SyncerSecret" + } + $url = "https://netperfapiwebapp.azurewebsites.net" + try { + $Response = Invoke-WebRequest -Uri "$url/getkeyvalue?key=$RunId" -Headers $headers + } catch { + Write-Host "Unable to fetch state. Creating a new one now." + $state = [pscustomobject]@{ + value=[pscustomobject]@{ + "SeqNum" = 0 + "Commands" = @($Command) + }} + $StateJson = $state | ConvertTo-Json + $Response = Invoke-WebRequest -Uri "$url/setkeyvalue?key=$RunId" -Headers $headers -Method Post -Body $StateJson -ContentType "application/json" + if ($Response.StatusCode -ne 200) { + Write-GHError "[Start-Remote-Passive] Failed to set the key value!" + throw "Failed to set the key value!" + } + return + } + $CurrState = $Response.Content | ConvertFrom-Json + $CurrState.Commands += $Command + $CurrState = [pscustomobject]@{ + value=$CurrState + } + $StateJson = $CurrState | ConvertTo-Json + $Response = Invoke-WebRequest -Uri "$url/setkeyvalue?key=$RunId" -Headers $headers -Method Post -Body $StateJson -ContentType "application/json" + if ($Response.StatusCode -ne 200) { + Write-GHError "[Start-Remote-Passive] Failed to set the key value!" + throw "Failed to set the key value!" + } + return + } + + Invoke-Command -Session $Session -ScriptBlock { + if (!(Test-Path $Using:RemoteStateDir)) { + New-Item -ItemType Directory $Using:RemoteStateDir | Out-Null + } + # Fetch all files names inside the directory + $files = Get-ChildItem $Using:RemoteStateDir + # Find the highest lexicographically sorted file name + $max = 0 + foreach ($file in $files) { + # Remove .ps1 extension from file.Name + $filename = $file.Name.split(".")[0] + $num = [int]($filename -replace "[^0-9]", "") + if ($num -gt $max) { + $max = $num + } + } + $newmax = $max + 1 + # Create a new file with the next number + Write-Host "Creating new file execute_$newmax.ps1" + New-Item -ItemType File (Join-Path $Using:RemoteStateDir ("execute_$newmax.ps1")) | Out-Null + $newFile = Join-Path $Using:RemoteStateDir ("execute_$newmax.ps1") + Set-Content -Path $newFile -Value $Using:Command + } +} + # Sends a special UDP packet to tell the remote secnetperf to shutdown, and then # waits for the job to complete. Finally, it returns the console output of the # job. @@ -306,6 +371,83 @@ function Stop-RemoteServer { return $RemoteResult -join "`n" } +function Stop-RemoteServerAsyncAwait { + param ($Session, $RemoteStateDir, $RemoteAddress, $RunId, $SyncerSecret) + # Ping side-channel socket on 9999 to tell the app to die + $Socket = New-Object System.Net.Sockets.UDPClient + $BytesToSend = @( + 0x57, 0xe6, 0x15, 0xff, 0x26, 0x4f, 0x0e, 0x57, + 0x88, 0xab, 0x07, 0x96, 0xb2, 0x58, 0xd1, 0x1c + ) + if ($Session -eq "NOT_SUPPORTED") { + for ($i = 0; $i -lt 30; $i++) { + $Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteAddress, 9999) | Out-Null + Start-Sleep -Seconds 8 | Out-Null + $headers = @{ + "secret" = "$SyncerSecret" + } + $url = "https://netperfapiwebapp.azurewebsites.net" + $Response = Invoke-WebRequest -Uri "$url/getkeyvalue?key=$RunId" -Headers $headers + if (!($Response.StatusCode -eq 200)) { + Write-GHError "[Stop-Remote-Passive] Failed to get the key value!" + throw "Failed to get the key value!" + } + $CurrState = $Response.Content | ConvertFrom-Json + if ($CurrState.SeqNum -eq $CurrState.Commands.Count) { + return + } + } + Write-GHError "[Stop-Remote-Passive] SeqNum less than Commands Count!" + throw "Unable to stop the remote server in time!" + } + $done = $false + $MaxSeqNum = 0 + for ($i = 0; $i -lt 30; $i++) { + $Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteAddress, 9999) | Out-Null + Start-Sleep -Seconds 5 | Out-Null + $files = Invoke-Command -Session $Session -ScriptBlock { Get-ChildItem $Using:RemoteStateDir } + foreach ($file in $files) { + $filename = $file.Name.split(".")[0] + $num = [int]($filename -replace "[^0-9]", "") + if ($num -gt $MaxSeqNum) { + $MaxSeqNum = $num + } + } + # Check if completed_.txt exists + foreach ($file in $files) { + if ($file.Name -eq "completed_$MaxSeqNum.txt") { + $done = $true + break + } + } + if ($done) { + break + } + } + if (!$done) { + Write-GHError "[Stop-Remote-Passive] Unable to find a corresponding 'completed_$MaxSeqNum' file!" + throw "Unable to stop the remote server in time!" + } +} + +function Wait-StartRemoteServerPassive { + param ($FullPath, $RemoteName, $OutputDir) + + for ($i = 0; $i -lt 30; $i++) { + Start-Sleep -Seconds 5 | Out-Null + Write-Host "Attempt $i to start the remote server, command: $FullPath -target:$RemoteName" + $Process = Start-LocalTest $FullPath "-target:$RemoteName" $OutputDir + $ConsoleOutput = Wait-LocalTest $Process $OutputDir $false 30000 $true + Write-Host "Wait-StartRemoteServerPassive: $ConsoleOutput" + $DidMatch = $ConsoleOutput -match "Completed" # Look for the special string to indicate success. + if ($DidMatch) { + return + } + } + + throw "Unable to start the remote server in time!" +} + # Creates a new local process to asynchronously run the test. function Start-LocalTest { param ($FullPath, $FullArgs, $OutputDir, $UseSudo) @@ -337,7 +479,7 @@ function Start-LocalTest { # Waits for a local test process to complete, and then returns the console output. function Wait-LocalTest { - param ($Process, $OutputDir, $testKernel, $TimeoutMs) + param ($Process, $OutputDir, $testKernel, $TimeoutMs, $Silent = $false) $StdOut = $Process.StandardOutput.ReadToEndAsync() $StdError = $Process.StandardError.ReadToEndAsync() # Wait for the process to exit. @@ -351,6 +493,10 @@ function Wait-LocalTest { $Out = $StdOut.Result.Trim() if ($Out.Length -ne 0) { Write-Host $Out } } catch {} + if ($Silent) { + Write-Host "Silently ignoring Client timeout!" + return "" + } throw "secnetperf: Client timed out!" } # Verify the process cleanly exitted. @@ -360,15 +506,27 @@ function Wait-LocalTest { $Out = $StdOut.Result.Trim() if ($Out.Length -ne 0) { Write-Host $Out } } catch {} + if ($Silent) { + Write-Host "Silently ignoring Client exit code: $($Process.ExitCode)" + return "" + } throw "secnetperf: Nonzero exit code: $($Process.ExitCode)" } # Wait for the output streams to flush. [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError)) $consoleTxt = $StdOut.Result.Trim() if ($consoleTxt.Length -eq 0) { + if ($Silent) { + Write-Host "Silently ignoring Client no console output!" + return "" + } throw "secnetperf: No console output (possibly crashed)!" } if ($consoleTxt.Contains("Error")) { + if ($Silent) { + Write-Host "Silently ignoring Client error: $($consoleTxt)" + return "" + } throw "secnetperf: $($consoleTxt.Substring(7))" # Skip over the "Error: " prefix } return $consoleTxt @@ -460,7 +618,7 @@ function Get-LatencyOutput { # Invokes secnetperf with the given arguments for both TCP and QUIC. function Invoke-Secnetperf { - param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter) + param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret) $values = @(@(), @()) $latency = $null @@ -485,7 +643,7 @@ function Invoke-Secnetperf { $execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode $clientPath = Repo-Path $SecNetPerfPath $serverArgs = "$execMode -io:$io" - $clientArgs = "-target:netperf-peer $ExeArgs -tcp:$tcp -trimout -watchdog:25000" + $clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000" if ($io -eq "xdp" -or $io -eq "qtip") { $serverArgs += " -pollidle:10000" $clientArgs += " -pollidle:10000" @@ -524,15 +682,17 @@ function Invoke-Secnetperf { $artifactDir = Repo-Path "artifacts/logs/$artifactName" $remoteArtifactDir = "$RemoteDir/artifacts/logs/$artifactName" New-Item -ItemType Directory $artifactDir -ErrorAction Ignore | Out-Null - Invoke-Command -Session $Session -ScriptBlock { - New-Item -ItemType Directory $Using:remoteArtifactDir -ErrorAction Ignore | Out-Null + if (!($Session -eq "NOT_SUPPORTED")) { + Invoke-Command -Session $Session -ScriptBlock { + New-Item -ItemType Directory $Using:remoteArtifactDir -ErrorAction Ignore | Out-Null + } } $clientOut = (Join-Path $artifactDir "client.console.log") $serverOut = (Join-Path $artifactDir "server.console.log") # Start logging on both sides, if configured. - if ($LogProfile -ne "" -and $LogProfile -ne "NULL") { + if ($LogProfile -ne "" -and $LogProfile -ne "NULL" -and !($Session -eq "NOT_SUPPORTED")) { Invoke-Command -Session $Session -ScriptBlock { try { & "$Using:RemoteDir/scripts/log.ps1" -Cancel } catch {} # Cancel any previous logging & "$Using:RemoteDir/scripts/log.ps1" -Start -Profile $Using:LogProfile -ProfileInScriptDirectory @@ -547,7 +707,17 @@ function Invoke-Secnetperf { # Start the server running. "> secnetperf $serverArgs" | Add-Content $serverOut - $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo + + $StateDir = "C:/_state" + if (!$IsWindows) { + $StateDir = "/etc/_state" + } + if ($Environment -eq "azure") { + Start-RemoteServerPassive $Session "$sudo$RemoteDir/$SecNetPerfPath $serverArgs" $StateDir $RunId $SyncerSecret + Wait-StartRemoteServerPassive "$sudo$clientPath" $RemoteName $artifactDir + } else { + $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo + } # Run the test multiple times, failing (for now) only if all tries fail. # TODO: Once all failures have been fixed, consider all errors fatal. @@ -586,7 +756,11 @@ function Invoke-Secnetperf { $testFailures = $true } finally { # Stop the server. - try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { } + if ($Environment -eq "azure") { + Stop-RemoteServerAsyncAwait $Session $StateDir $RemoteName $RunId $SyncerSecret + } else { + try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { } + } # Stop any logging and copy the logs to the artifacts folder. if ($LogProfile -ne "" -and $LogProfile -ne "NULL") { diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 592e96041c..3e12c20bab 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -71,7 +71,16 @@ param ( [string]$RemoteName = "netperf-peer", [Parameter(Mandatory = $false)] - [string]$UserName = "secnetperf" + [string]$UserName = "secnetperf", + + [Parameter(Mandatory = $false)] + [string]$RemotePowershellSupported = "TRUE", + + [Parameter(Mandatory = $false)] + [string]$RunId = "0", + + [Parameter(Mandatory = $false)] + [string]$SyncerSecret = "0" ) Set-StrictMode -Version "Latest" @@ -86,6 +95,7 @@ if (!$isWindows) { $RemoteDir = "/home/$UserName/_work/quic" } } + $SecNetPerfDir = "artifacts/bin/$plat/$($arch)_Release_$tls" $SecNetPerfPath = "$SecNetPerfDir/secnetperf" if ($io -eq "") { @@ -102,33 +112,57 @@ if ($isWindows -and $NoLogs) { } $useXDP = ($io -eq "xdp" -or $io -eq "qtip") -# Set up the connection to the peer over remote powershell. -Write-Host "Connecting to $RemoteName" -$Attempts = 0 -while ($Attempts -lt 5) { - try { - if ($isWindows) { - $username = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultUserName - $password = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultPassword | ConvertTo-SecureString -AsPlainText -Force - $cred = New-Object System.Management.Automation.PSCredential ($username, $password) - $Session = New-PSSession -ComputerName $RemoteName -Credential $cred -ConfigurationName PowerShell.7 - } else { - $Session = New-PSSession -HostName $RemoteName -UserName $UserName -SSHTransport +if ($RemotePowershellSupported -eq "TRUE") { + + # Set up the connection to the peer over remote powershell. + Write-Host "Connecting to $RemoteName" + $Attempts = 0 + while ($Attempts -lt 5) { + if ($environment -eq "azure") { + if ($isWindows) { + Write-Host "Attempting to connect..." + $Session = New-PSSession -ComputerName $RemoteName -ConfigurationName PowerShell.7 + break + } else { + # On Azure in 1ES Linux environments, remote powershell is not supported (yet). + $Session = "NOT_SUPPORTED" + Write-Host "Remote PowerShell is not supported in Azure 1ES Linux environments" + break + } + } + try { + if ($isWindows) { + $username = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultUserName + $password = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultPassword | ConvertTo-SecureString -AsPlainText -Force + $cred = New-Object System.Management.Automation.PSCredential ($username, $password) + $Session = New-PSSession -ComputerName $RemoteName -Credential $cred -ConfigurationName PowerShell.7 + } else { + $Session = New-PSSession -HostName $RemoteName -UserName $UserName -SSHTransport + } + break + } catch { + Write-Host "Error $_" + $Attempts += 1 + Start-Sleep -Seconds 10 } - break - } catch { - $Attempts += 1 - Start-Sleep -Seconds 10 } -} -if ($null -eq $Session) { - Write-GHError "Failed to create remote session" - exit 1 + if ($null -eq $Session) { + Write-GHError "Failed to create remote session" + exit 1 + } + +} else { + $Session = "NOT_SUPPORTED" + Write-Host "Remote PowerShell is not supported in this environment" } -# Make sure nothing is running from a previous run. -Cleanup-State $Session $RemoteDir +if (!($environment -eq "azure") -and !($Session -eq "NOT_SUPPORTED")) { + # Make sure nothing is running from a previous run. This only applies to non-azure / 1ES environments. + Write-Host "NOT RUNNING ON AZURE AND POWERSHELL SUPPORTED" + Write-Host "Session: $Session, $(!($Session -eq "NOT_SUPPORTED"))" + Cleanup-State $Session $RemoteDir +} # Create intermediary files. New-Item -ItemType File -Name "latency.txt" @@ -145,26 +179,29 @@ if ($io -eq "wsk") { Remove-Item -Force -Recurse $KernelDir | Out-Null } -# Copy the artifacts to the peer. -Write-Host "Copying files to peer" -Invoke-Command -Session $Session -ScriptBlock { - if (Test-Path $Using:RemoteDir) { - Remove-Item -Force -Recurse $Using:RemoteDir | Out-Null + +if (!($Session -eq "NOT_SUPPORTED")) { + # Copy the artifacts to the peer. + Write-Host "Copying files to peer" + Invoke-Command -Session $Session -ScriptBlock { + if (Test-Path $Using:RemoteDir) { + Remove-Item -Force -Recurse $Using:RemoteDir | Out-Null + } + New-Item -ItemType Directory -Path $Using:RemoteDir -Force | Out-Null + } + Copy-Item -ToSession $Session ./artifacts -Destination "$RemoteDir/artifacts" -Recurse + Copy-Item -ToSession $Session ./scripts -Destination "$RemoteDir/scripts" -Recurse + Copy-Item -ToSession $Session ./src/manifest/MsQuic.wprp -Destination "$RemoteDir/scripts" + + # Create the logs directories on both machines. + New-Item -ItemType Directory -Path ./artifacts/logs | Out-Null + Invoke-Command -Session $Session -ScriptBlock { + New-Item -ItemType Directory -Path $Using:RemoteDir/artifacts/logs | Out-Null } - New-Item -ItemType Directory -Path $Using:RemoteDir -Force | Out-Null -} -Copy-Item -ToSession $Session ./artifacts -Destination "$RemoteDir/artifacts" -Recurse -Copy-Item -ToSession $Session ./scripts -Destination "$RemoteDir/scripts" -Recurse -Copy-Item -ToSession $Session ./src/manifest/MsQuic.wprp -Destination "$RemoteDir/scripts" - -# Create the logs directories on both machines. -New-Item -ItemType Directory -Path ./artifacts/logs | Out-Null -Invoke-Command -Session $Session -ScriptBlock { - New-Item -ItemType Directory -Path $Using:RemoteDir/artifacts/logs | Out-Null } # Collect some info about machine state. -if (!$NoLogs -and $isWindows) { +if (!$NoLogs -and $isWindows -and !($Session -eq "NOT_SUPPORTED")) { $Arguments = "-SkipNetsh" if (Get-Help Get-NetView -Parameter SkipWindowsRegistry -ErrorAction Ignore) { $Arguments += " -SkipWindowsRegistry" @@ -224,7 +261,7 @@ $json["run_args"] = $allTests try { # Prepare the machines for the testing. -if ($isWindows) { +if ($isWindows -and !($environment -eq "azure")) { Write-Host "Preparing local machine for testing" ./scripts/prepare-machine.ps1 -ForTest -InstallSigningCertificates @@ -238,12 +275,14 @@ if ($isWindows) { if (!$HasTestSigning) { Write-Host "Test Signing Not Enabled!" } } -# Configure the dump collection. -Configure-DumpCollection $Session +if (!($Session -eq "NOT_SUPPORTED")) { + # Configure the dump collection. + Configure-DumpCollection $Session +} # Install any dependent drivers. -if ($useXDP -and $isWindows) { Install-XDP $Session $RemoteDir } -if ($io -eq "wsk") { Install-Kernel $Session $RemoteDir $SecNetPerfDir } +if ($useXDP -and $isWindows -and !($Session -eq "NOT_SUPPORTED")) { Install-XDP $Session $RemoteDir } +if ($io -eq "wsk" -and !($Session -eq "NOT_SUPPORTED")) { Install-Kernel $Session $RemoteDir $SecNetPerfDir } if (!$isWindows) { # Make sure the secnetperf binary is executable. @@ -252,11 +291,13 @@ if (!$isWindows) { if ($io -eq "xdp") { $GRO = "off" } - Invoke-Command -Session $Session -ScriptBlock { - $env:LD_LIBRARY_PATH = "${env:LD_LIBRARY_PATH}:$Using:RemoteDir/$Using:SecNetPerfDir" - chmod +x "$Using:RemoteDir/$Using:SecNetPerfPath" - if ($Using:os -eq "ubuntu-22.04") { - sudo sh -c "ethtool -K eth0 generic-receive-offload $Using:GRO" + if (!($Session -eq "NOT_SUPPORTED")) { + Invoke-Command -Session $Session -ScriptBlock { + $env:LD_LIBRARY_PATH = "${env:LD_LIBRARY_PATH}:$Using:RemoteDir/$Using:SecNetPerfDir" + chmod +x "$Using:RemoteDir/$Using:SecNetPerfPath" + if ($Using:os -eq "ubuntu-22.04") { + sudo sh -c "ethtool -K eth0 generic-receive-offload $Using:GRO" + } } } $fullPath = Repo-Path $SecNetPerfDir @@ -292,7 +333,7 @@ $regressionJson = Get-Content -Raw -Path "watermark_regression.json" | ConvertFr Write-Host "Setup complete! Running all tests" foreach ($testId in $allTests.Keys) { $ExeArgs = $allTests[$testId] + " -io:$io" - $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $testId $ExeArgs $io $filter + $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $testId $ExeArgs $io $filter $environment $RunId $SyncerSecret $Test = $Output[-1] if ($Test.HasFailures) { $hasFailures = $true } @@ -323,7 +364,12 @@ Write-Host "Tests complete!" } finally { # Perform any necessary cleanup. - try { Cleanup-State $Session $RemoteDir } catch { } + try { + if ($Session -eq "NOT_SUPPORTED") { + throw "Cleanup not needed" + } + Cleanup-State $Session $RemoteDir + } catch { } try { if (Get-ChildItem -Path ./artifacts/logs -File -Recurse) {