Add infrastructure to enable 2-machine network testing on 1ES Azure pools.

scripts/secnetperf-helpers.psm1

@@ -283,6 +283,71 @@ function Start-RemoteServer {
     throw "Server failed to start!"
 }
 
+# Passively starts the server on the remote machine by queuing up a new script to execute.
+function Start-RemoteServerPassive {
+    param ($Session, $Command, $RemoteStateDir, $RunId, $SyncerSecret)
+    if ($Session -eq "NOT_SUPPORTED") {
+        Write-Host "Command to start server: $Command"
+        $headers = @{
+            "secret" = "$SyncerSecret"
+        }
+        $url = "https://netperfapiwebapp.azurewebsites.net"
+        try {
+            $Response = Invoke-WebRequest -Uri "$url/getkeyvalue?key=$RunId" -Headers $headers
+        } catch {
+            Write-Host "Unable to fetch state. Creating a new one now."
+            $state = [pscustomobject]@{
+                value=[pscustomobject]@{
+                "SeqNum" = 0
+                "Commands" = @($Command)
+            }}
+            $StateJson = $state | ConvertTo-Json
+            $Response = Invoke-WebRequest -Uri "$url/setkeyvalue?key=$RunId" -Headers $headers -Method Post -Body $StateJson -ContentType "application/json"
+            if ($Response.StatusCode -ne 200) {
+                Write-GHError "[Start-Remote-Passive] Failed to set the key value!"
+                throw "Failed to set the key value!"
+            }
+            return
+        }
+        $CurrState = $Response.Content | ConvertFrom-Json
+        $CurrState.Commands += $Command
+        $CurrState = [pscustomobject]@{
+            value=$CurrState
+        }
+        $StateJson = $CurrState | ConvertTo-Json
+        $Response = Invoke-WebRequest -Uri "$url/setkeyvalue?key=$RunId" -Headers $headers -Method Post -Body $StateJson -ContentType "application/json"
+        if ($Response.StatusCode -ne 200) {
+            Write-GHError "[Start-Remote-Passive] Failed to set the key value!"
+            throw "Failed to set the key value!"
+        }
+        return
+    }
+
+    Invoke-Command -Session $Session -ScriptBlock {
+        if (!(Test-Path $Using:RemoteStateDir)) {
+            New-Item -ItemType Directory $Using:RemoteStateDir | Out-Null
+        }
+        # Fetch all files names inside the directory
+        $files = Get-ChildItem $Using:RemoteStateDir
+        # Find the highest lexicographically sorted file name
+        $max = 0
+        foreach ($file in $files) {
+            # Remove .ps1 extension from file.Name
+            $filename = $file.Name.split(".")[0]
+            $num = [int]($filename -replace "[^0-9]", "")
+            if ($num -gt $max) {
+                $max = $num
+            }
+        }
+        $newmax = $max + 1
+        # Create a new file with the next number
+        Write-Host "Creating new file execute_$newmax.ps1"
+        New-Item -ItemType File (Join-Path $Using:RemoteStateDir ("execute_$newmax.ps1")) | Out-Null
+        $newFile = Join-Path $Using:RemoteStateDir ("execute_$newmax.ps1")
+        Set-Content -Path $newFile -Value $Using:Command
+    }
+}
+
 # Sends a special UDP packet to tell the remote secnetperf to shutdown, and then
 # waits for the job to complete. Finally, it returns the console output of the
 # job.
@@ -306,6 +371,83 @@ function Stop-RemoteServer {
     return $RemoteResult -join "`n"
 }
 
+function Stop-RemoteServerAsyncAwait {
+    param ($Session, $RemoteStateDir, $RemoteAddress, $RunId, $SyncerSecret)
+    # Ping side-channel socket on 9999 to tell the app to die
+    $Socket = New-Object System.Net.Sockets.UDPClient
+    $BytesToSend = @(
+        0x57, 0xe6, 0x15, 0xff, 0x26, 0x4f, 0x0e, 0x57,
+        0x88, 0xab, 0x07, 0x96, 0xb2, 0x58, 0xd1, 0x1c
+    )
+    if ($Session -eq "NOT_SUPPORTED") {
+        for ($i = 0; $i -lt 30; $i++) {
+            $Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteAddress, 9999) | Out-Null
+            Start-Sleep -Seconds 8 | Out-Null
+            $headers = @{
+                "secret" = "$SyncerSecret"
+            }
+            $url = "https://netperfapiwebapp.azurewebsites.net"
+            $Response = Invoke-WebRequest -Uri "$url/getkeyvalue?key=$RunId" -Headers $headers
+            if (!($Response.StatusCode -eq 200)) {
+                Write-GHError "[Stop-Remote-Passive] Failed to get the key value!"
+                throw "Failed to get the key value!"
+            }
+            $CurrState = $Response.Content | ConvertFrom-Json
+            if ($CurrState.SeqNum -eq $CurrState.Commands.Count) {
+                return
+            }
+        }
+        Write-GHError "[Stop-Remote-Passive] SeqNum less than Commands Count!"
+        throw "Unable to stop the remote server in time!"
+    }
+    $done = $false
+    $MaxSeqNum = 0
+    for ($i = 0; $i -lt 30; $i++) {
+        $Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteAddress, 9999) | Out-Null
+        Start-Sleep -Seconds 5 | Out-Null
+        $files = Invoke-Command -Session $Session -ScriptBlock { Get-ChildItem $Using:RemoteStateDir }
+        foreach ($file in $files) {
+            $filename = $file.Name.split(".")[0]
+            $num = [int]($filename -replace "[^0-9]", "")
+            if ($num -gt $MaxSeqNum) {
+                $MaxSeqNum = $num
+            }
+        }
+        # Check if completed_<MaxSeqNum>.txt exists
+        foreach ($file in $files) {
+            if ($file.Name -eq "completed_$MaxSeqNum.txt") {
+                $done = $true
+                break
+            }
+        }
+        if ($done) {
+            break
+        }
+    }
+    if (!$done) {
+        Write-GHError "[Stop-Remote-Passive] Unable to find a corresponding 'completed_$MaxSeqNum' file!"
+        throw "Unable to stop the remote server in time!"
+    }
+}
+
+function Wait-StartRemoteServerPassive {
+    param ($FullPath, $RemoteName, $OutputDir)
+
+    for ($i = 0; $i -lt 30; $i++) {
+        Start-Sleep -Seconds 5 | Out-Null
+        Write-Host "Attempt $i to start the remote server, command: $FullPath -target:$RemoteName"
+        $Process = Start-LocalTest $FullPath "-target:$RemoteName" $OutputDir
+        $ConsoleOutput = Wait-LocalTest $Process $OutputDir $false 30000 $true
+        Write-Host "Wait-StartRemoteServerPassive: $ConsoleOutput"
+        $DidMatch = $ConsoleOutput -match "Completed" # Look for the special string to indicate success.
+        if ($DidMatch) {
+            return
+        }
+    }
+
+    throw "Unable to start the remote server in time!"
+}
+
 # Creates a new local process to asynchronously run the test.
 function Start-LocalTest {
     param ($FullPath, $FullArgs, $OutputDir, $UseSudo)
@@ -337,7 +479,7 @@ function Start-LocalTest {
 
 # Waits for a local test process to complete, and then returns the console output.
 function Wait-LocalTest {
-    param ($Process, $OutputDir, $testKernel, $TimeoutMs)
+    param ($Process, $OutputDir, $testKernel, $TimeoutMs, $Silent = $false)
     $StdOut = $Process.StandardOutput.ReadToEndAsync()
     $StdError = $Process.StandardError.ReadToEndAsync()
     # Wait for the process to exit.
@@ -351,6 +493,10 @@ function Wait-LocalTest {
             $Out = $StdOut.Result.Trim()
             if ($Out.Length -ne 0) { Write-Host $Out }
         } catch {}
+        if ($Silent) {
+            Write-Host "Silently ignoring Client timeout!"
+            return ""
+        }
         throw "secnetperf: Client timed out!"
     }
     # Verify the process cleanly exitted.
@@ -360,15 +506,27 @@ function Wait-LocalTest {
             $Out = $StdOut.Result.Trim()
             if ($Out.Length -ne 0) { Write-Host $Out }
         } catch {}
+        if ($Silent) {
+            Write-Host "Silently ignoring Client exit code: $($Process.ExitCode)"
+            return ""
+        }
         throw "secnetperf: Nonzero exit code: $($Process.ExitCode)"
     }
     # Wait for the output streams to flush.
     [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError))
     $consoleTxt = $StdOut.Result.Trim()
     if ($consoleTxt.Length -eq 0) {
+        if ($Silent) {
+            Write-Host "Silently ignoring Client no console output!"
+            return ""
+        }
         throw "secnetperf: No console output (possibly crashed)!"
     }
     if ($consoleTxt.Contains("Error")) {
+        if ($Silent) {
+            Write-Host "Silently ignoring Client error: $($consoleTxt)"
+            return ""
+        }
         throw "secnetperf: $($consoleTxt.Substring(7))" # Skip over the "Error: " prefix
     }
     return $consoleTxt
@@ -460,7 +618,7 @@ function Get-LatencyOutput {
 
 # Invokes secnetperf with the given arguments for both TCP and QUIC.
 function Invoke-Secnetperf {
-    param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter)
+    param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret)
 
     $values = @(@(), @())
     $latency = $null
@@ -485,7 +643,7 @@ function Invoke-Secnetperf {
     $execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode
     $clientPath = Repo-Path $SecNetPerfPath
     $serverArgs = "$execMode -io:$io"
-    $clientArgs = "-target:netperf-peer $ExeArgs -tcp:$tcp -trimout -watchdog:25000"
+    $clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000"
     if ($io -eq "xdp" -or $io -eq "qtip") {
         $serverArgs += " -pollidle:10000"
         $clientArgs += " -pollidle:10000"
@@ -524,15 +682,17 @@ function Invoke-Secnetperf {
     $artifactDir = Repo-Path "artifacts/logs/$artifactName"
     $remoteArtifactDir = "$RemoteDir/artifacts/logs/$artifactName"
     New-Item -ItemType Directory $artifactDir -ErrorAction Ignore | Out-Null
-    Invoke-Command -Session $Session -ScriptBlock {
-        New-Item -ItemType Directory $Using:remoteArtifactDir -ErrorAction Ignore | Out-Null
+    if (!($Session -eq "NOT_SUPPORTED")) {
+        Invoke-Command -Session $Session -ScriptBlock {
+            New-Item -ItemType Directory $Using:remoteArtifactDir -ErrorAction Ignore | Out-Null
+        }
     }
 
     $clientOut = (Join-Path $artifactDir "client.console.log")
     $serverOut = (Join-Path $artifactDir "server.console.log")
 
     # Start logging on both sides, if configured.
-    if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {
+    if ($LogProfile -ne "" -and $LogProfile -ne "NULL" -and !($Session -eq "NOT_SUPPORTED")) {
         Invoke-Command -Session $Session -ScriptBlock {
             try { & "$Using:RemoteDir/scripts/log.ps1" -Cancel } catch {} # Cancel any previous logging
             & "$Using:RemoteDir/scripts/log.ps1" -Start -Profile $Using:LogProfile -ProfileInScriptDirectory
@@ -547,7 +707,17 @@ function Invoke-Secnetperf {
 
     # Start the server running.
     "> secnetperf $serverArgs" | Add-Content $serverOut
-    $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo
+
+    $StateDir = "C:/_state"
+    if (!$IsWindows) {
+        $StateDir = "/etc/_state"
+    }
+    if ($Environment -eq "azure") {
+        Start-RemoteServerPassive $Session "$sudo$RemoteDir/$SecNetPerfPath $serverArgs" $StateDir $RunId $SyncerSecret
+        Wait-StartRemoteServerPassive "$sudo$clientPath" $RemoteName $artifactDir
+    } else {
+        $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo
+    }
 
     # Run the test multiple times, failing (for now) only if all tries fail.
     # TODO: Once all failures have been fixed, consider all errors fatal.
@@ -586,7 +756,11 @@ function Invoke-Secnetperf {
         $testFailures = $true
     } finally {
         # Stop the server.
-        try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { }
+        if ($Environment -eq "azure") {
+            Stop-RemoteServerAsyncAwait $Session $StateDir $RemoteName $RunId $SyncerSecret
+        } else {
+            try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { }
+        }
 
         # Stop any logging and copy the logs to the artifacts folder.
         if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {