[main] Update dependencies from dotnet/arcade (#30002)

[main] Update dependencies from dotnet/arcade


 - Workaround CS0165 error

- despite calling `SetupAsync()`, compiler thought `adding` was uninitialized

 - Merge branch 'main' into darc-main-a5fd6730-bd72-4e81-b34b-d8112480f819

 - Merge branch 'main' into darc-main-a5fd6730-bd72-4e81-b34b-d8112480f819

 - Download the 7.0.3 runtime
- not carried in 8.0.x .NET SDKs

 - !fixup! Correct nesting in global.json
- nit: put the "sdk" section at the top of the file

 - !fixup! Some tests need an ASP.NET runtime
- add the v7.0.3 runtime to global.json
- nit: clean up console logging about a missing variable

 - !fixup! Correct typo in my nit fix

azure-pipelines.yml

@@ -120,6 +120,8 @@ stages:
                 - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }}
                 # Default expects tsaoptions.json under SourceRoot.
                 - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json'
+              - ${{ else }}:
+                - _AdditionalBuildArgs: ''
             steps:
               - task: NuGetCommand@2
                 displayName: 'Clear NuGet caches'

eng/Version.Details.xml

@@ -39,17 +39,17 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.22630.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23110.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e82404fca08383513e0b0b3c5308d4a9b18b7c7a</Sha>
+      <Sha>96d8be74c39a4765ec919ff9bebf9e0c875fc195</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="8.0.0-beta.22630.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Templating" Version="8.0.0-beta.23110.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e82404fca08383513e0b0b3c5308d4a9b18b7c7a</Sha>
+      <Sha>96d8be74c39a4765ec919ff9bebf9e0c875fc195</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.22630.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.23110.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e82404fca08383513e0b0b3c5308d4a9b18b7c7a</Sha>
+      <Sha>96d8be74c39a4765ec919ff9bebf9e0c875fc195</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -26,7 +26,7 @@
     <MicrosoftExtensionsLoggingVersion>8.0.0-preview.2.23105.4</MicrosoftExtensionsLoggingVersion>
   </PropertyGroup>
   <PropertyGroup Label="Dependencies from dotnet/arcade">
-    <MicrosoftDotNetBuildTasksTemplatingVersion>8.0.0-beta.22630.1</MicrosoftDotNetBuildTasksTemplatingVersion>
+    <MicrosoftDotNetBuildTasksTemplatingVersion>8.0.0-beta.23110.3</MicrosoftDotNetBuildTasksTemplatingVersion>
   </PropertyGroup>
   <PropertyGroup Label="Other dependencies">
     <MicrosoftCodeAnalysisVersion>4.2.0</MicrosoftCodeAnalysisVersion>

eng/common/cross/build-rootfs.sh

@@ -48,12 +48,14 @@ __UbuntuPackages+=" symlinks"
 __UbuntuPackages+=" libicu-dev"
 __UbuntuPackages+=" liblttng-ust-dev"
 __UbuntuPackages+=" libunwind8-dev"
+__UbuntuPackages+=" libnuma-dev"
 
 __AlpinePackages+=" gettext-dev"
 __AlpinePackages+=" icu-dev"
 __AlpinePackages+=" libunwind-dev"
 __AlpinePackages+=" lttng-ust-dev"
 __AlpinePackages+=" compiler-rt-static"
+__AlpinePackages+=" numactl-dev"
 
 # runtime libraries' dependencies
 __UbuntuPackages+=" libcurl4-openssl-dev"
@@ -147,17 +149,17 @@ while :; do
             __BuildArch=ppc64le
             __UbuntuArch=ppc64el
             __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
+            __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp5/}"
             unset __LLDB_Package
             ;;
         riscv64)
             __BuildArch=riscv64
             __UbuntuArch=riscv64
             __UbuntuRepo="http://deb.debian.org/debian-ports"
             __CodeName=sid
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+            __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
             unset __LLDB_Package
 
             if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then
@@ -168,9 +170,9 @@ while :; do
             __BuildArch=s390x
             __UbuntuArch=s390x
             __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
-            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
+            __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
+            __UbuntuPackages="${__UbuntuPackages// libomp5/}"
             unset __LLDB_Package
             ;;
         x64)
@@ -310,6 +312,8 @@ done
 
 if [[ "$__BuildArch" == "armel" ]]; then
     __LLDB_Package="lldb-3.5-dev"
+elif [[ "$__BuildArch" == "arm" && "$__AlpineVersion" == "3.13" ]]; then
+    __AlpinePackages="${__AlpinePackages//numactl-dev/}"
 fi
 
 __UbuntuPackages+=" ${__LLDB_Package:-}"

eng/common/generate-locproject.ps1

@@ -34,6 +34,25 @@ $jsonTemplateFiles | ForEach-Object {
 $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
 $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
+if (-not $wxlFiles) {
+    $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } #  pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
+    if ($wxlEnFiles) {
+      $wxlFiles = @()
+      $wxlEnFiles | ForEach-Object {
+        $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+        $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+      }
+    }
+}
+
+$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files
+$macosHtmlFiles = @()
+if ($macosHtmlEnFiles) {
+    $macosHtmlEnFiles | ForEach-Object {
+        $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+        $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+    }
+}
 
 $xlfFiles = @()
 
@@ -99,8 +118,7 @@ $locJson = @{
                     $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
                     $continue = $true
                     foreach ($exclusion in $exclusions.Exclusions) {
-                        if ($_.FullName.Contains($exclusion))
-                        {
+                        if ($_.FullName.Contains($exclusion)) {
                             $continue = $false
                         }
                     }
@@ -115,6 +133,30 @@ $locJson = @{
                     }
                 }
             )
+        },
+        @{
+            LanguageSet = $LanguageSet
+            CloneLanguageSet = "VS_macOS_CloneLanguages"
+            LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" )
+            LocItems = @(
+                $macosHtmlFiles | ForEach-Object {
+                    $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($_.FullName.Contains($exclusion)) {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if ($continue) {
+                        return @{
+                            SourceFile = $sourceFile
+                            CopyOption = "LangIDOnPath"
+                            OutputPath = $outputPath
+                        }
+                    }
+                }
+            )
         }
     )
 }

eng/common/init-tools-native.ps1

@@ -83,7 +83,8 @@ try {
                     Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
   if ($NativeTools) {
     if ($PathPromotion -eq $True) {
-      if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build
+      $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools"
+      if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine
         $NativeTools.PSObject.Properties | ForEach-Object {
           $ToolName = $_.Name
           $ToolVersion = $_.Value
@@ -93,11 +94,6 @@ try {
             if ($ToolVersion -eq "latest") {
               $ToolVersion = ""
             }
-            $ArcadeToolsDirectory = "C:\arcade-tools"
-            if (-not (Test-Path $ArcadeToolsDirectory)) {
-              Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed."
-              exit 1
-            }
             $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)
             if ($ToolDirectories -eq $null) {
               Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
@@ -125,6 +121,7 @@ try {
 
           if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
             Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding."
+            Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image."
           }
         }
         exit 0

eng/common/native/init-compiler.sh

@@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then
     # Set default versions
     if [ -z "$majorVersion" ]; then
         # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
-        if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
+        if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
         elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi
 
         for version in $versions; do

eng/common/templates/job/job.yml

@@ -88,6 +88,16 @@ jobs:
     - ${{ if ne(variable.group, '') }}:
       - group: ${{ variable.group }}
 
+    # handle template variable syntax
+    # example:
+    # - template: path/to/template.yml
+    #   parameters:
+    #     [key]: [value]
+    - ${{ if ne(variable.template, '') }}:
+      - template: ${{ variable.template }}
+        ${{ if ne(variable.parameters, '') }}:
+          parameters: ${{ variable.parameters }}
+
     # handle key-value variable syntax.
     # example:
     # - [key]: [value]
@@ -124,7 +134,7 @@ jobs:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@0
 
-  - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+  - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}:
     - task: DownloadPipelineArtifact@2
       inputs:
         buildType: current
@@ -161,7 +171,7 @@ jobs:
           TeamName: $(_TeamName)
 
   - ${{ if ne(parameters.artifacts.publish, '') }}:
-    - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+    - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
       - task: CopyFiles@2
         displayName: Gather binaries for publish to artifacts
         inputs:
@@ -182,7 +192,7 @@ jobs:
           ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
         continueOnError: true
         condition: always()
-    - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+    - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
       - publish: artifacts/log
         artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
         displayName: Publish logs

eng/common/templates/job/source-build.yml

@@ -46,20 +46,12 @@ jobs:
     # source-build builds run in Docker, including the default managed platform.
     # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic
     pool:
-      # Main environments
-      ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}:
-        name: NetCore-Public
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
-      ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}:
-        name: NetCore1ESPool-Internal
-        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
 
-      # Servicing build environments
-      ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}:
-        name: NetCore-Svc-Public
-        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
-      ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}:
-        name: NetCore1ESPool-Svc-Internal
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
 
   ${{ if ne(parameters.platform.pool, '') }}:

eng/common/templates/job/source-index-stage1.yml

@@ -1,6 +1,6 @@
 parameters:
   runAsPublic: false
-  sourceIndexPackageVersion: 1.0.1-20220804.1
+  sourceIndexPackageVersion: 1.0.1-20221220.2
   sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
@@ -40,10 +40,10 @@ jobs:
     - ${{ preStep }}
 
   - task: UseDotNet@2
-    displayName: Use .NET Core sdk 3.1
+    displayName: Use .NET Core SDK 6
     inputs:
       packageType: sdk
-      version: 3.1.x
+      version: 6.0.x
       installationPath: $(Agent.TempDirectory)/dotnet
       workingDirectory: $(Agent.TempDirectory)
 

eng/common/templates/variables/pool-providers.yml

@@ -16,7 +16,7 @@
 #  First, import the template in an arcade-ified repo to pick up the variables, e.g.:
 #
 #  variables:
-#  - template: eng/common/templates/variables/pool-providers.yml
+#  - template: /eng/common/templates/variables/pool-providers.yml
 #
 #  ... then anywhere specifying the pool provider use the runtime variables,
 #      $(DncEngInternalBuildPool) and $  (DncEngPublicBuildPool), e.g.:
@@ -26,23 +26,32 @@
 #           demands: ImageOverride -equals windows.vs2019.amd64
 
 variables:
-# Coalesce the target and source branches so we know when a PR targets a release branch
-# If these variables are somehow missing, fall back to main (tends to have more capacity)
-- name: BranchNameForPoolSelection
-  value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }}
+  # Coalesce the target and source branches so we know when a PR targets a release branch
+  # If these variables are somehow missing, fall back to main (tends to have more capacity)
 
-# Any new -Svc alternative pools should have variables added here to allow for splitting work
-
-# Main branch pools
-- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}:
+  # Any new -Svc alternative pools should have variables added here to allow for splitting work
   - name: DncEngPublicBuildPool
-    value: NetCore-Public
-  - name: DncEngInternalBuildPool
-    value: NetCore1ESPool-Internal
+    value: $[
+        replace(
+          replace(
+            eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+            True,
+            'NetCore-Svc-Public'
+          ),
+          False,
+          'NetCore-Public'
+        )
+      ]
 
-# Release branch pools
-- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}:
-  - name: DncEngPublicBuildPool
-    value: NetCore-Svc-Public
   - name: DncEngInternalBuildPool
-    value: NetCore1ESPool-Svc-Internal
+    value: $[
+        replace(
+          replace(
+            eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+            True,
+            'NetCore1ESPool-Svc-Internal'
+          ),
+          False,
+          'NetCore1ESPool-Internal'
+        )
+      ]

eng/common/tools.ps1

@@ -581,7 +581,7 @@ function InitializeBuildTool() {
       ExitWithExitCode 1
     }
     $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
-    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net7.0' }
+    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' }
   } elseif ($msbuildEngine -eq "vs") {
     try {
       $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -743,6 +743,8 @@ function MSBuild() {
       (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
       (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
       (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
+      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
     )
     $selectedPath = $null
     foreach ($path in $possiblePaths) {

eng/common/tools.sh

@@ -312,7 +312,7 @@ function InitializeBuildTool {
   # return values
   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
   _InitializeBuildToolCommand="msbuild"
-  _InitializeBuildToolFramework="net7.0"
+  _InitializeBuildToolFramework="net8.0"
 }
 
 # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
@@ -428,6 +428,8 @@ function MSBuild {
     possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
     possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" )
     possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+    possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" )
+    possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" )
     for path in "${possiblePaths[@]}"; do
       if [[ -f $path ]]; then
         selectedPath=$path