[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.19470.9">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20074.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>84778cd659cc0509e9ebf651c73f2341ae82df8c</Sha>
+      <Sha>841c83d8a5b8d8a6bd427f6798b086780138aa9a</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -1,7 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
   <PropertyGroup>
-    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
     <!-- This repo version -->
     <VersionPrefix>3.2.0</VersionPrefix>
     <NugetPackagePrefix>1.0.1</NugetPackagePrefix>
@@ -58,12 +57,4 @@
     <RoslynDiagnosticsAnalyzersVersion>2.6.2-beta2</RoslynDiagnosticsAnalyzersVersion>
     <StyleCopAnalyzersVersion>1.2.0-beta.66</StyleCopAnalyzersVersion>
   </PropertyGroup>
-  <PropertyGroup>
-    <RestoreSources>
-      $(RestoreSources);
-      https://dotnet.myget.org/F/roslyn/api/v3/index.json;
-      https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json;
-      https://api.nuget.org/v3/index.json
-    </RestoreSources>
-  </PropertyGroup>
 </Project>

eng/common/CheckSymbols.ps1

@@ -5,11 +5,12 @@ param(
 )
 
 Add-Type -AssemblyName System.IO.Compression.FileSystem
+. $PSScriptRoot\pipeline-logging-functions.ps1
 
 function FirstMatchingSymbolDescriptionOrDefault {
   param( 
     [string] $FullPath,                  # Full path to the module that has to be checked
-    [string] $TargetServerParam,         # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+    [string] $TargetServerParameter,         # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
     [string] $SymbolsPath
   )
 
@@ -21,36 +22,36 @@ function FirstMatchingSymbolDescriptionOrDefault {
   # checking and which type of file was uploaded.
 
   # The file itself is returned
-  $SymbolPath = $SymbolsPath + "\" + $FileName
+  $SymbolPath = $SymbolsPath + '\' + $FileName
 
   # PDB file for the module
-  $PdbPath = $SymbolPath.Replace($Extension, ".pdb")
+  $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
 
   # PDB file for R2R module (created by crossgen)
-  $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb")
+  $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
 
   # DBG file for a .so library
-  $SODbg = $SymbolPath.Replace($Extension, ".so.dbg")
+  $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
 
   # DWARF file for a .dylib
-  $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
+  $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
 
-  .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
+  .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParameter $FullPath -o $SymbolsPath | Out-Null
 
   if (Test-Path $PdbPath) {
-    return "PDB"
+    return 'PDB'
   }
   elseif (Test-Path $NGenPdb) {
-    return "NGen PDB"
+    return 'NGen PDB'
   }
   elseif (Test-Path $SODbg) {
-    return "DBG for SO"
+    return 'DBG for SO'
   }  
   elseif (Test-Path $DylibDwarf) {
-    return "Dwarf for Dylib"
+    return 'Dwarf for Dylib'
   }  
   elseif (Test-Path $SymbolPath) {
-    return "Module"
+    return 'Module'
   }
   else {
     return $null
@@ -68,15 +69,15 @@ function CountMissingSymbols {
   }
 
   # Extensions for which we'll look for symbols
-  $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib")
+  $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
 
   # How many files are missing symbol information
   $MissingSymbols = 0
 
   $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
   $PackageGuid = New-Guid
   $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
-  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols"
+  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
 
   [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
 
@@ -86,31 +87,31 @@ function CountMissingSymbols {
   Get-ChildItem -Recurse $ExtractPath |
     Where-Object {$RelevantExtensions -contains $_.Extension} |
     ForEach-Object {
-      if ($_.FullName -Match "\\ref\\") {
+      if ($_.FullName -Match '\\ref\\') {
         Write-Host "`t Ignoring reference assembly file" $_.FullName
         return
       }
 
-      $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath
-      $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath
+      $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName  -TargetServerParameter '--microsoft-symbol-server' -SymbolsPath $SymbolsPath
+      $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--internal-server' -SymbolsPath $SymbolsPath
 
       Write-Host -NoNewLine "`t Checking file" $_.FullName "... "
 
       if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
-        Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")"
+        Write-Host "Symbols found on MSDL (${$SymbolsOnMSDL}) and SymWeb (${$SymbolsOnSymWeb})"
       }
       else {
         $MissingSymbols++
 
         if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
-          Write-Host "No symbols found on MSDL or SymWeb!"
+          Write-Host 'No symbols found on MSDL or SymWeb!'
         }
         else {
           if ($SymbolsOnMSDL -eq $null) {
-            Write-Host "No symbols found on MSDL!"
+            Write-Host 'No symbols found on MSDL!'
           }
           else {
-            Write-Host "No symbols found on SymWeb!"
+            Write-Host 'No symbols found on SymWeb!'
           }
         }
       }
@@ -129,26 +130,26 @@ function CheckSymbolsAvailable {
   Get-ChildItem "$InputPath\*.nupkg" |
     ForEach-Object {
       $FileName = $_.Name
-	  
+
       # These packages from Arcade-Services include some native libraries that
       # our current symbol uploader can't handle. Below is a workaround until
       # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
-      if ($FileName -Match "Microsoft\.DotNet\.Darc\.") {
+      if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
         Write-Host "Ignoring Arcade-services file: $FileName"
         Write-Host
         return
       }
-      elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") {
+      elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
         Write-Host "Ignoring Arcade-services file: $FileName"
         Write-Host
         return
       }
-	  
+
       Write-Host "Validating $FileName "
       $Status = CountMissingSymbols "$InputPath\$FileName"
 
       if ($Status -ne 0) {
-        Write-Error "Missing symbols for $Status modules in the package $FileName"
+        Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
       }
 
       Write-Host

eng/common/PublishToSymbolServers.proj

@@ -37,6 +37,8 @@
     <PropertyGroup>
       <DotNetSymbolExpirationInDays Condition="'$(DotNetSymbolExpirationInDays)' == ''">3650</DotNetSymbolExpirationInDays>
       <PublishToSymbolServer>true</PublishToSymbolServer>
+      <PublishToSymWeb Condition="'$(PublishToSymWeb)' == ''">true</PublishToSymWeb>
+      <PublishToMSDL Condition="'$(PublishToMSDL)' == ''">true</PublishToMSDL>
       <PublishToSymbolServer Condition="'@(FilesToPublishToSymbolServer)' == '' and '@(PackagesToPublishToSymbolServer)' == ''">false</PublishToSymbolServer>
     </PropertyGroup>
 
@@ -56,7 +58,7 @@
                     DryRun="false"
                     ConvertPortablePdbsToWindowsPdbs="false"
                     PdbConversionTreatAsWarning=""
-                    Condition="$(PublishToSymbolServer)"/>
+                    Condition="$(PublishToSymbolServer) and $(PublishToMSDL)"/>
 
     <!-- 
       Symbol Uploader: SymWeb 
@@ -73,7 +75,7 @@
                     DryRun="false"
                     ConvertPortablePdbsToWindowsPdbs="false"
                     PdbConversionTreatAsWarning=""
-                    Condition="$(PublishToSymbolServer)"/>
+                    Condition="$(PublishToSymbolServer) and $(PublishToSymWeb)"/>
   </Target>
 
   <ItemGroup>

eng/common/SetupNugetSources.ps1

@@ -0,0 +1,141 @@
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+#  - task: PowerShell@2
+#    displayName: Setup Private Feeds Credentials
+#    condition: eq(variables['Agent.OS'], 'Windows_NT')
+#    inputs:
+#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+#    env:
+#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory = $true)][string]$ConfigFile,
+    [Parameter(Mandatory = $true)][string]$Password
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+. $PSScriptRoot\tools.ps1
+
+# Add source entry to PackageSources
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+    $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
+
+    if ($packageSource -eq $null)
+    {
+        $packageSource = $doc.CreateElement("add")
+        $packageSource.SetAttribute("key", $SourceName)
+        $packageSource.SetAttribute("value", $SourceEndPoint)
+        $sources.AppendChild($packageSource) | Out-Null
+    }
+    else {
+        Write-Host "Package source $SourceName already present."
+    }
+
+    AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+}
+
+# Add a credential node for the specified source
+function AddCredential($creds, $source, $username, $password) {
+    # Looks for credential configuration for the given SourceName. Create it if none is found.
+    $sourceElement = $creds.SelectSingleNode($Source)
+    if ($sourceElement -eq $null)
+    {
+        $sourceElement = $doc.CreateElement($Source)
+        $creds.AppendChild($sourceElement) | Out-Null
+    }
+
+    # Add the <Username> node to the credential if none is found.
+    $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
+    if ($usernameElement -eq $null)
+    {
+        $usernameElement = $doc.CreateElement("add")
+        $usernameElement.SetAttribute("key", "Username")
+        $sourceElement.AppendChild($usernameElement) | Out-Null
+    }
+    $usernameElement.SetAttribute("value", $Username)
+
+    # Add the <ClearTextPassword> to the credential if none is found.
+    # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
+    #   -> https://github.com/NuGet/Home/issues/5526
+    $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
+    if ($passwordElement -eq $null)
+    {
+        $passwordElement = $doc.CreateElement("add")
+        $passwordElement.SetAttribute("key", "ClearTextPassword")
+        $sourceElement.AppendChild($passwordElement) | Out-Null
+    }
+    $passwordElement.SetAttribute("value", $Password)
+}
+
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
+    $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+
+    Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
+
+    ForEach ($PackageSource in $maestroPrivateSources) {
+        Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
+        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+    }
+}
+
+if (!(Test-Path $ConfigFile -PathType Leaf)) {
+  Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+  ExitWithExitCode 1
+}
+
+if (!$Password) {
+    Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
+    ExitWithExitCode 1
+}
+
+# Load NuGet.config
+$doc = New-Object System.Xml.XmlDocument
+$filename = (Get-Item $ConfigFile).FullName
+$doc.Load($filename)
+
+# Get reference to <PackageSources> or create one if none exist already
+$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
+if ($sources -eq $null) {
+    $sources = $doc.CreateElement("packageSources")
+    $doc.DocumentElement.AppendChild($sources) | Out-Null
+}
+
+# Looks for a <PackageSourceCredentials> node. Create it if none is found.
+$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+if ($creds -eq $null) {
+    $creds = $doc.CreateElement("packageSourceCredentials")
+    $doc.DocumentElement.AppendChild($creds) | Out-Null
+}
+
+# Insert credential nodes for Maestro's private feeds
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
+
+$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']")
+if ($dotnet3Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+}
+
+$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
+if ($dotnet31Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+}
+
+$doc.Save($filename)