Merge pull request #17 from bflattened/merge-release/7.0

Merge release/7.0
bflattened · Sep 2, 2022 · dd433f6 · dd433f6
2 parents ab6e957 + 0281ed7
commit dd433f6
Show file tree

Hide file tree

Showing 532 changed files with 16,351 additions and 13,484 deletions.
diff --git a/docs/design/features/OsrDetailsAndDebugging.md b/docs/design/features/OsrDetailsAndDebugging.md
@@ -338,7 +338,7 @@ Note if a Tier0 method is recursive and has loops there can be some interesting
 
 ### Seeing which OSR methods are created
 
-* `DOTNET_DumpJittedMethods=1` will specially mark OSR methods with the inspiring IL offsets.
+* `DOTNET_JitDisasmSummary=1` will specially mark OSR methods with the inspiring IL offsets.
 
 For example, running a libraries test with some stressful OSR settings, there ended up being 699 OSR methods jitted out of 160675 total methods. Grepping for OSR in the dump output, the last few lines were:
 

diff --git a/docs/workflow/testing/testing-workloads.md b/docs/workflow/testing/testing-workloads.md
@@ -2,7 +2,7 @@
 
 Workloads based on packages in `artifacts` can be installed, and used for testing.
 
-- This is done by installing a specified SDK version (`$(SdkVersionForWorkloadTesting)`) in `artifacts/bin/dotnet-workload`.
+- This is done by installing a specified SDK version (`$(SdkVersionForWorkloadTesting)`) in `artifacts/bin/dotnet-net7`.
 - Then the manifest for the workload in `@(WorkloadIdForTesting)` is installed
     - Then workload packs are installed
     - packs, or manifests not generated by `runtime`, are restored from nuget

diff --git a/eng/Signing.props b/eng/Signing.props
@@ -1,5 +1,4 @@
-<Project InitialTargets="SetupFilesToSign">
-
+<Project>
   <PropertyGroup>
     <!--
       Windows arm/arm64 jobs don't have MSIs to sign. Keep it simple: allow not finding any matches
@@ -20,6 +19,13 @@
     <!-- apphost and comhost template files are not signed, by design. -->
     <FileSignInfo Include="apphost.exe;singlefilehost.exe;comhost.dll" CertificateName="None" />
 
+    <!--
+      The DAC and the DBI must go through special signing provisioning using a system separate
+      from MicroBuild.
+    -->
+    <FileSignInfo Include="mscordaccore.dll" CertificateName="None" />
+    <FileSignInfo Include="mscordbi.dll" CertificateName="None" />
+
     <!-- We don't need to code sign .js files because they are not used in Windows Script Host. -->
     <!-- WARNING: Needs to happed outside of any target -->
     <FileExtensionSignInfo Update=".js" CertificateName="None" />
@@ -31,8 +37,6 @@
     <FileSignInfo Include="Mono.Cecil.Pdb.dll" CertificateName="3PartySHA2" />
     <FileSignInfo Include="Mono.Cecil.Rocks.dll" CertificateName="3PartySHA2" />
 
-    <FileSignInfo Include="mscordaccore.dll" CertificateName="MicrosoftSHA2" />
-
     <!-- Exclude symbol packages from have a NuGet signature. These are never pushed to NuGet.org or
          other feeds (in fact, that have identical identity to their non-symbol variant) -->
     <DownloadedSymbolPackages Include="$(DownloadDirectory)**\*.symbols.nupkg" />
@@ -61,33 +65,4 @@
     <ItemsToSignPostBuild Include="@(ItemsToSignWithoutPaths->Distinct())" Condition="'$(PostBuildSign)' == 'true'" />
     <ItemsToSign Include="@(ItemsToSignWithPaths->Distinct())" Condition="'$(PostBuildSign)' != 'true'" />
   </ItemGroup>
-
-  <Target Name="SetupFilesToSign">
-    <!-- Ensure that we don't miss the DAC or DBI with the globbing below -->
-    <PropertyGroup Condition="'$(SignDiagnostics)' == 'true' or '$(SignDiagnosticsPackages)' == 'true'">
-      <AllowEmptySignList>false</AllowEmptySignList>
-    </PropertyGroup>
-
-    <ItemGroup Condition="'$(SignDiagnostics)' == 'true'">
-      <ItemsToSign Include="$(DiagnosticsFilesRoot)\**\mscordaccore*.dll" />
-      <ItemsToSign Include="$(DiagnosticsFilesRoot)\**\mscordbi.dll" />
-      <!--
-        The DAC should be signed with the SHA2 cert (both long and short name).
-        We already add the short-name DAC above, so add the long-name DAC here.
-      -->
-      <DacFileSignInfo Include="@(ItemsToSign->'%(FileName)%(Extension)')"
-                    Condition="$([System.String]::new('%(FileName)').StartsWith('mscordaccore'))" />
-      <FileSignInfo Include="@(DacFileSignInfo->ClearMetadata()->Distinct())"
-                    Exclude="mscordaccore.dll"
-                    CertificateName="MicrosoftSHA2" />
-    </ItemGroup>
-
-    <ItemGroup Condition="'$(SignDiagnosticsPackages)' == 'true'">
-      <!-- The cross OS diagnostics symbol packages need to be signed as they are the only packages
-      that have a specific version of assets that are only meant to be indexed in symbol servers.
-      Since only *symbols.nupkg get indexed, and installer doesn't produce these, we need to glob them for signing. -->
-      <ItemsToSign Include="$(PackagesFolder)\**\*CrossOsDiag*.nupkg" />
-    </ItemGroup>
-  </Target>
-
 </Project>
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
diff --git a/eng/Versions.props b/eng/Versions.props
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
@@ -76,10 +76,10 @@ __FreeBSDPackages+=" openssl"
 __FreeBSDPackages+=" krb5"
 __FreeBSDPackages+=" terminfo-db"
 
-__IllumosPackages="icu-64.2nb2"
-__IllumosPackages+=" mit-krb5-1.16.2nb4"
-__IllumosPackages+=" openssl-1.1.1e"
-__IllumosPackages+=" zlib-1.2.11"
+__IllumosPackages="icu"
+__IllumosPackages+=" mit-krb5"
+__IllumosPackages+=" openssl"
+__IllumosPackages+=" zlib"
 
 __HaikuPackages="gmp"
 __HaikuPackages+=" gmp_devel"
@@ -390,14 +390,18 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     if [[ "$__UseMirror" == 1 ]]; then
         BaseUrl=http://pkgsrc.smartos.skylime.net
     fi
-    BaseUrl="$BaseUrl/packages/SmartOS/2020Q1/${__illumosArch}/All"
+    BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
+    echo "Downloading manifest"
+    wget "$BaseUrl"
     echo "Downloading dependencies."
     read -ra array <<<"$__IllumosPackages"
     for package in "${array[@]}"; do
-       echo "Installing $package..."
+        echo "Installing '$package'"
+        package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')"
+        echo "Resolved name '$package'"
         wget "$BaseUrl"/"$package".tgz
         ar -x "$package".tgz
-        tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null
+        tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null
     done
     echo "Cleaning up temporary files."
     popd

diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
@@ -33,6 +33,8 @@ $jsonTemplateFiles | ForEach-Object {
 
 $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
+$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
+
 $xlfFiles = @()
 
 $allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
@@ -77,8 +79,7 @@ $locJson = @{
                                 CopyOption = "LangIDOnPath"
                                 OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
                             }
-                        }
-                        else {
+                        } else {
                             return @{
                                 SourceFile = $sourceFile
                                 CopyOption = "LangIDOnName"
@@ -88,6 +89,32 @@ $locJson = @{
                     }
                 }
             )
+        },
+        @{
+            CloneLanguageSet = "WiX_CloneLanguages"
+            LssFiles = @( "wxl_loc.lss" )
+            LocItems = @(
+                $wxlFiles | ForEach-Object {
+                    $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($outputPath.Contains($exclusion))
+                        {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if ($continue)
+                    {
+                        return @{
+                            SourceFile = $sourceFile
+                            CopyOption = "LangIDOnPath"
+                            OutputPath = $outputPath
+                            Languages = "cs-CZ;de-DE;es-ES;fr-FR;it-IT;ja-JP;ko-KR;pl-PL;pt-BR;ru-RU;tr-TR;zh-CN;zh-TW"
+                        }
+                    }
+                }
+            )
         }
     )
 }

diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.1.0" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.2.1" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1
@@ -0,0 +1,37 @@
+
+function Install-Gdn {
+    param(
+        [string]$Path,
+
+        # If omitted, install the latest version of Guardian, otherwise install that specific version.
+        [string]$Version
+    )
+
+    $ErrorActionPreference = 'Stop'
+    Set-StrictMode -Version 2.0
+    $disableConfigureToolsetImport = $true
+    $global:LASTEXITCODE = 0
+
+    # `tools.ps1` checks $ci to perform some actions. Since the SDL
+    # scripts don't necessarily execute in the same agent that run the
+    # build.ps1/sh script this variable isn't automatically set.
+    $ci = $true
+    . $PSScriptRoot\..\tools.ps1
+
+    $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
+
+    if ($Version) {
+        $argumentList += "-Version $Version"
+    }
+
+    Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
+
+    $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path
+
+    if (!$gdnCliPath)
+    {
+        Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian'
+    }
+
+    return $gdnCliPath.FullName
+}
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
@@ -46,7 +46,7 @@ jobs:
     # source-build builds run in Docker, including the default managed platform.
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: NetCore1ESPool-Public
+        name: NetCore-Public
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: NetCore1ESPool-Internal

diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
@@ -28,7 +28,7 @@ jobs:
   ${{ if eq(parameters.pool, '') }}:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: NetCore1ESPool-Public
+        name: NetCore-Public
         demands: ImageOverride -equals windows.vs2019.amd64.open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: NetCore1ESPool-Internal

diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml
@@ -8,29 +8,26 @@ parameters:
   condition: ''
 
 steps:
-- ${{ if ne(parameters.overrideGuardianVersion, '') }}:
-  - powershell: |
-      $content = Get-Content $(GuardianPackagesConfigFile)
-
-      Write-Host "packages.config content was:`n$content"
-
-      $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
-      $content | Set-Content $(GuardianPackagesConfigFile)
-
-      Write-Host "packages.config content updated to:`n$content"
-    displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
+- task: NuGetAuthenticate@1
+  inputs:
+    nuGetServiceConnections: GuardianConnect
 
 - task: NuGetToolInstaller@1
   displayName: 'Install NuGet.exe'
 
-- task: NuGetCommand@2
-  displayName: 'Install Guardian'
-  inputs:
-    restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
-    feedsToUse: config
-    nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
-    externalFeedCredentials: GuardianConnect
-    restoreDirectory: $(Build.SourcesDirectory)\.packages
+- ${{ if ne(parameters.overrideGuardianVersion, '') }}:
+  - pwsh: |
+      . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1
+      $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }}
+      Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
+    displayName: Install Guardian (Overridden)
+
+- ${{ if eq(parameters.overrideGuardianVersion, '') }}:
+  - pwsh: |
+      . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1
+      $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts
+      Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
+    displayName: Install Guardian
 
 - ${{ if ne(parameters.overrideParameters, '') }}:
   - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
@@ -40,7 +37,7 @@ steps:
 
 - ${{ if eq(parameters.overrideParameters, '') }}:
   - powershell: ${{ parameters.executeAllSdlToolsScript }}
-      -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion)
+      -GuardianCliLocation $(GuardianCliLocation)
       -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
       -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
       ${{ parameters.additionalParameters }}

diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -365,8 +365,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
 
   # If the version of msbuild is going to be xcopied,
   # use this version. Version matches a package here:
-  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.1.0&view=overview
-  $defaultXCopyMSBuildVersion = '17.1.0'
+  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.2.1&view=overview
+  $defaultXCopyMSBuildVersion = '17.2.1'
 
   if (!$vsRequirements) {
     if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {

diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets
@@ -179,7 +179,6 @@
       <LibrariesRuntimeFiles Condition="'$(TargetOS)' == 'Browser'"
                              Include="
         $(LibrariesNativeArtifactsPath)dotnet.js;
-        $(LibrariesNativeArtifactsPath)src\dotnet-crypto-worker.js;
         $(LibrariesNativeArtifactsPath)dotnet.d.ts;
         $(LibrariesNativeArtifactsPath)dotnet-legacy.d.ts;
         $(LibrariesNativeArtifactsPath)package.json;

diff --git a/eng/native/configureplatform.cmake b/eng/native/configureplatform.cmake
@@ -2,7 +2,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/functions.cmake)
 
 # If set, indicates that this is not an officially supported release.
 # Release branches should set this to false.
-set(PRERELEASE 1)
+set(PRERELEASE 0)
 
 #----------------------------------------
 # Detect and set platform variable names

diff --git a/eng/nativepgo.targets b/eng/nativepgo.targets
@@ -1,7 +1,7 @@
 <Project>
   <PropertyGroup>
-    <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'windows' And ('$(TargetArchitecture)' == 'x64' Or '$(TargetArchitecture)' == 'x86')">true</NativeOptimizationDataSupported>
-    <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'Linux' And '$(TargetArchitecture)' == 'x64'">true</NativeOptimizationDataSupported>
+    <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'windows' And ('$(TargetArchitecture)' == 'x64' Or '$(TargetArchitecture)' == 'x86' Or '$(TargetArchitecture)' == 'arm64')">true</NativeOptimizationDataSupported>
+    <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'Linux' And ('$(TargetArchitecture)' == 'x64' Or '$(TargetArchitecture)' == 'arm64')">true</NativeOptimizationDataSupported>
     <NativeOptimizationDataSupported Condition="'$(NoPgoOptimize)' == 'true'">false</NativeOptimizationDataSupported>
     <NativeOptimizationDataSupported Condition="'$(Configuration)' != 'Release'">false</NativeOptimizationDataSupported>
 
@@ -16,14 +16,20 @@
     <PackageReference Include="optimization.windows_nt-x86.PGO.CoreCLR"
       Version="$(optimizationPGOCoreCLRVersion)"
       GeneratePathProperty="true" />
+    <PackageReference Include="optimization.windows_nt-arm64.PGO.CoreCLR"
+      Version="$(optimizationPGOCoreCLRVersion)"
+      GeneratePathProperty="true" />
     <PackageReference Include="optimization.linux-x64.PGO.CoreCLR"
       Version="$(optimizationPGOCoreCLRVersion)"
       GeneratePathProperty="true" />
+    <PackageReference Include="optimization.linux-arm64.PGO.CoreCLR"
+      Version="$(optimizationPGOCoreCLRVersion)"
+      GeneratePathProperty="true" />
   </ItemGroup>
 
 
   <!--                                                                       -->
-  <!-- Task: GetPgoDataPackagePath                                          -->
+  <!-- Task: GetPgoDataPackagePath                                           -->
   <!--                                                                       -->
   <!-- Notes:                                                                -->
   <!--                                                                       -->

diff --git a/eng/pipelines/common/evaluate-default-paths.yml b/eng/pipelines/common/evaluate-default-paths.yml
@@ -113,7 +113,8 @@ jobs:
       - src/mono/wasm/host/*
       - src/mono/wasm/runtime/*
       - src/mono/wasm/templates/*
-      - src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/*
+      - src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.net6.Manifest/*
+      - src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.net7.Manifest/*
       - src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/*
       - src/mono/nuget/Microsoft.NET.Runtime.wasm.Sample.Mono/*
       - src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/*

diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml
@@ -135,12 +135,12 @@ jobs:
       pool:
         # Public Linux Build Pool
         ${{ if and(or(in(parameters.osGroup, 'Linux', 'FreeBSD', 'Android', 'Tizen'), eq(parameters.jobParameters.hostedOs, 'Linux')), eq(variables['System.TeamProject'], 'public')) }}:
-          name:  NetCore1ESPool-Public
+          name:  NetCore-Svc-Public
           demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
 
         # Official Build Linux Pool
         ${{ if and(or(in(parameters.osGroup, 'Linux', 'FreeBSD', 'Browser', 'Android', 'Tizen'), eq(parameters.jobParameters.hostedOs, 'Linux')), ne(variables['System.TeamProject'], 'public')) }}:
-          name: NetCore1ESPool-Internal
+          name: NetCore1ESPool-Svc-Internal
           demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
 
         # OSX Build Pool (we don't have on-prem OSX BuildPool
@@ -149,12 +149,12 @@ jobs:
 
         # Official Build Windows Pool
         ${{ if and(eq(parameters.osGroup, 'windows'), ne(variables['System.TeamProject'], 'public')) }}:
-          name: NetCore1ESPool-Internal
+          name: NetCore1ESPool-Svc-Internal
           demands: ImageOverride -equals windows.vs2022.amd64
 
         # Public Windows Build Pool
         ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), eq(variables['System.TeamProject'], 'public')) }}:
-          name: NetCore1ESPool-Public
+          name: NetCore-Svc-Public
           demands: ImageOverride -equals windows.vs2022.amd64.open