Implement PGO in pipelines for AMD64 architecture; supply training test scenarios

.github/actions/spelling/expect/web.txt

@@ -15,5 +15,4 @@ winui
 appshellintegration
 cppreference
 gfycat
-what3words
 Guake

NOTICE.md

@@ -252,10 +252,26 @@ DEALINGS IN THE SOFTWARE.
 
 ```
 
+# Microsoft Open Source
+
+This product also incorporates source code from other Microsoft open source projects, all licensed under the MIT license.
+
+## `GSL`
+
+**Source**: [https://github.com/microsoft/GSL](https://github.com/microsoft/GSL)
+
+## `Microsoft-UI-XAML`
+
+**Source**: [https://github.com/microsoft/Microsoft-UI-XAML](https://github.com/microsoft/Microsoft-UI-XAML)
+
 ## `VirtualDesktopUtils`
 
 **Source**: [https://github.com/microsoft/PowerToys](https://github.com/microsoft/PowerToys)
 
+## `wil`
+
+**Source**: [https://github.com/microsoft/wil](https://github.com/microsoft/wil)
+
 ### License
 
 ```

OpenConsole.sln

@@ -395,6 +395,11 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnitTests_Control", "src\ca
 		{CA5CAD1A-44BD-4AC7-AC72-6CA5B3AB89ED} = {CA5CAD1A-44BD-4AC7-AC72-6CA5B3AB89ED}
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowsTerminal.UIA.Tests", "src\cascadia\WindowsTerminal_UIATests\WindowsTerminal.UIA.Tests.csproj", "{F19DACD5-0C6E-40DC-B6E4-767A3200542C}"
+	ProjectSection(ProjectDependencies) = postProject
+		{CA5CAD1A-224A-4171-B13A-F16E576FDD12} = {CA5CAD1A-224A-4171-B13A-F16E576FDD12}
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		AuditMode|Any CPU = AuditMode|Any CPU
@@ -3256,6 +3261,54 @@ Global
 		{C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Release|x64.Build.0 = Release|x64
 		{C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Release|x86.ActiveCfg = Release|Win32
 		{C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B}.Release|x86.Build.0 = Release|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|Any CPU.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|Any CPU.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|ARM.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|ARM.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|ARM64.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|ARM64.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|DotNet_x64Test.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|DotNet_x64Test.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|DotNet_x86Test.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|DotNet_x86Test.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|x64.ActiveCfg = Debug|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|x64.Build.0 = Debug|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|x86.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.AuditMode|x86.Build.0 = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|ARM.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|ARM64.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|DotNet_x64Test.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|DotNet_x86Test.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|x64.ActiveCfg = Debug|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|x64.Build.0 = Debug|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|x86.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Debug|x86.Build.0 = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|Any CPU.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|Any CPU.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|ARM.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|ARM.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|ARM64.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|ARM64.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|DotNet_x64Test.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|DotNet_x64Test.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|DotNet_x86Test.ActiveCfg = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|DotNet_x86Test.Build.0 = Debug|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|x64.ActiveCfg = Debug|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|x64.Build.0 = Debug|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|x86.ActiveCfg = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Fuzzing|x86.Build.0 = Debug|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|Any CPU.ActiveCfg = Release|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|ARM.ActiveCfg = Release|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|ARM64.ActiveCfg = Release|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|ARM64.Build.0 = Release|ARM64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|DotNet_x64Test.ActiveCfg = Release|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|DotNet_x86Test.ActiveCfg = Release|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|x64.ActiveCfg = Release|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|x64.Build.0 = Release|x64
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|x86.ActiveCfg = Release|Win32
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C}.Release|x86.Build.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -3353,6 +3406,7 @@ Global
 		{9921CA0A-320C-4460-8623-3A3196E7F4CB} = {59840756-302F-44DF-AA47-441A9D673202}
 		{05D9052F-D78F-478F-968A-2DE38A6DB996} = {E8F24881-5E37-4362-B191-A3BA0ED7F4EB}
 		{C323DAEE-B307-4C7B-ACE5-7293CBEFCB5B} = {BDB237B6-1D1D-400F-84CC-40A58FA59C8E}
+		{F19DACD5-0C6E-40DC-B6E4-767A3200542C} = {BDB237B6-1D1D-400F-84CC-40A58FA59C8E}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {3140B1B7-C8EE-43D1-A772-D82A7061A271}

build/Helix/AzurePipelinesHelperScripts.ps1

@@ -29,4 +29,147 @@ function GetQueryTestRunsUri
     $baseUri = GetAzureDevOpsBaseUri -CollectionUri $CollectionUri -TeamProject $TeamProject
     $queryUri = "$baseUri/_apis/test/runs?buildUri=$BuildUri$includeRunDetailsParameter&api-version=5.0"
     return $queryUri
+}
+
+function Get-HelixJobTypeFromTestRun
+{
+    Param ($testRun)
+
+    $testRunSingleResultUri = "$($testRun.url)/results?`$top=1&`$skip=0&api-version=5.1"
+    $singleTestResult = Invoke-RestMethod -Uri $testRunSingleResultUri -Method Get -Headers $azureDevOpsRestApiHeaders
+    $count = $singleTestResult.value.Length
+    if($count -eq 0)
+    {
+        # If the count is 0, then results have not yet been reported for this run.
+        # We only care about completed runs with results, so it is ok to just return 'UNKNOWN' for this run.
+        return "UNKNOWN"
+    }
+    else
+    {
+        $info = ConvertFrom-Json $singleTestResult.value.comment
+        $helixJobId = $info.HelixJobId
+        $job = Invoke-RestMethodWithRetries "https://helix.dot.net/api/2019-06-17/jobs/${helixJobId}?access_token=${HelixAccessToken}"
+        return $job.Type
+    }
+}
+
+function Append-HelixAccessTokenToUrl
+{
+    Param ([string]$url, [string]$token)
+    if($url.Contains("?"))
+    {
+        $url = "$($url)&access_token=$($token)"
+    }
+    else
+    {
+        $url = "$($url)?access_token=$($token)"
+    }
+    return $url
+}
+
+
+# The Helix Rest api is sometimes unreliable. So we call these apis with retry logic. 
+# Note: The Azure DevOps apis are stable and do not need to be called with this retry logic.
+$helixApiRetries = 0
+$helixApiRetriesMax = 10 
+
+function Download-StringWithRetries
+{
+    Param ([string]$fileName, [string]$url)
+
+    $result = ""
+    $done = $false
+
+    while(!($done))
+    {
+        try
+        {
+            Write-Host "Downloading $fileName"
+            $result = (New-Object System.Net.WebClient).DownloadString($url)
+            $done = $true
+        }
+        catch
+        {
+            Write-Host "Failed to download $fileName $($PSItem.Exception)"
+
+            $helixApiRetries = $helixApiRetries + 1
+            if($helixApiRetries -lt $helixApiRetriesMax)
+            {
+                Write-Host "Sleep and retry download of $fileName"
+                Start-Sleep 60
+            }
+            else
+            {
+                throw "Failed to download $fileName"
+            }
+        }
+    }
+
+    return $result
+}
+
+function Invoke-RestMethodWithRetries
+{
+    Param ([string]$url,$Headers)
+
+    $result = @()
+    $done = $false
+
+    while(!($done))
+    {
+        try
+        {
+            $result = Invoke-RestMethod -Uri $url -Method Get -Headers $Headers
+            $done = $true
+        }
+        catch
+        {
+            Write-Host "Failed to invoke Rest method $($PSItem.Exception)"
+
+            $helixApiRetries = $helixApiRetries + 1
+            if($helixApiRetries -lt $helixApiRetriesMax)
+            {
+                Write-Host "Sleep and retry invoke"
+                Start-Sleep 60
+            }
+            else
+            {
+                throw "Failed to invoke Rest method"
+            }
+        }
+    }
+
+    return $result
+}
+
+function Download-FileWithRetries
+{
+    Param ([string]$fileurl, [string]$destination)
+
+    $done = $false
+
+    while(!($done))
+    {
+        try
+        {
+            Write-Host "Downloading $destination"
+            $webClient.DownloadFile($fileurl, $destination)
+            $done = $true
+        }
+        catch
+        {
+            Write-Host "Failed to download $destination $($PSItem.Exception)"
+
+            $helixApiRetries = $helixApiRetries + 1
+            if($helixApiRetries -lt $helixApiRetriesMax)
+            {
+                Write-Host "Sleep and retry download of $destination"
+                Start-Sleep 60
+            }
+            else
+            {
+                throw "Failed to download $destination"
+            }
+        }
+    }
 }

build/Helix/PrepareHelixPayload.ps1

@@ -20,10 +20,12 @@ New-Item -ItemType Directory -Force -Path $payloadDir
 # Copy files from nuget packages
 Copy-Item "$nugetPackagesDir\microsoft.windows.apps.test.1.0.181203002\lib\netcoreapp2.1\*.dll" $payloadDir
 Copy-Item "$nugetPackagesDir\Microsoft.Taef.10.58.210305002\build\Binaries\$Platform\*" $payloadDir
-Copy-Item "$nugetPackagesDir\Microsoft.Taef.10.58.210305002\build\Binaries\$Platform\CoreClr\*" $payloadDir
+Copy-Item "$nugetPackagesDir\Microsoft.Taef.10.58.210305002\build\Binaries\$Platform\NetFx4.5\*" $payloadDir
 New-Item -ItemType Directory -Force -Path "$payloadDir\.NETCoreApp2.1\"
 Copy-Item "$nugetPackagesDir\runtime.win-$Platform.microsoft.netcore.app.2.1.0\runtimes\win-$Platform\lib\netcoreapp2.1\*" "$payloadDir\.NETCoreApp2.1\"
 Copy-Item "$nugetPackagesDir\runtime.win-$Platform.microsoft.netcore.app.2.1.0\runtimes\win-$Platform\native\*" "$payloadDir\.NETCoreApp2.1\"
+New-Item -ItemType Directory -Force -Path "$payloadDir\content\"
+Copy-Item "$nugetPackagesDir\Microsoft.Internal.Windows.Terminal.TestContent.1.0.0\content\*" "$payloadDir\content\"
 
 function Copy-If-Exists
 {
@@ -52,3 +54,13 @@ Copy-Item "build\helix\HelixTestHelpers.cs" "$payloadDir"
 Copy-Item "build\helix\runtests.cmd" $payloadDir
 Copy-Item "build\helix\InstallTestAppDependencies.ps1" "$payloadDir"
 Copy-Item "build\Helix\EnsureMachineState.ps1" "$payloadDir"
+
+# Copy the APPX package from the 'drop' artifact dir
+Copy-Item "$repoDirectory\Artifacts\$ArtifactName\appx\CascadiaPackage_0.0.1.0_$Platform.msix" $payloadDir\CascadiaPackage.zip
+
+# Rename it to extension of ZIP because Expand-Archive is real sassy on the build machines
+# and refuses to unzip it because of its file extension while on a desktop, it just 
+# does the job without complaining.
+
+# Extract the APPX package
+Expand-Archive -LiteralPath $payloadDir\CascadiaPackage.zip -DestinationPath $payloadDir\appx