From 25bfc0867f0efaebcc9ba0cfe0c1f03e0ea54852 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Thu, 1 Dec 2022 13:39:19 -0600 Subject: [PATCH 1/2] Updating 'Microsoft.DotNet.Arcade.Sdk': '5.0.0-beta.22276.2' => '5.0.0-beta.22526.12' --- eng/Version.Details.xml | 4 +- eng/Versions.props | 3 +- eng/common/build.ps1 | 5 + eng/common/darc-init.ps1 | 3 +- eng/common/init-tools-native.ps1 | 144 ++++++++++++------ eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 2 +- .../templates/post-build/post-build.yml | 9 +- eng/common/tools.ps1 | 4 + global.json | 2 +- 10 files changed, 124 insertions(+), 54 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 31eb326d23f..d1aa55e64dc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 9c6a04ea1e79e9fcd4e60abd5d2c577075787f93 + 7fafb6feb8f17f5dac9e8930c37016d250032c55 https://github.com/nuget/nuget.client diff --git a/eng/Versions.props b/eng/Versions.props index 8fc95605c2c..5a445f6014b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,7 +2,8 @@ - 16.11.3release + 16.11.3 + release 15.1.0.0 preview true diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c0817e..678e9b20eb7 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -25,6 +25,7 @@ Param( [switch] $prepareMachine, [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', + [switch] $nativeToolsOnMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -144,6 +146,9 @@ try { $nodeReuse = $false } + if ($nativeToolsOnMachine) { + $env:NativeToolsOnMachine = $true + } if ($restore) { InitializeNativeTools } diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 435e7641341..7df4726cb26 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -10,8 +10,7 @@ param ( function InstallDarcCli ($darcVersion, $toolpath) { $darcCliPackageName = 'microsoft.dotnet.darc' - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" + $dotnet = "dotnet" $toolList = & "$dotnet" tool list -g if ($toolList -like "*$darcCliPackageName*") { diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00a6f..fbc67effc36 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,102 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false + if ($PathPromotion -eq $True) { + if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ArcadeToolsDirectory = "C:\arcade-tools" + if (-not (Test-Path $ArcadeToolsDirectory)) { + Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." + exit 1 } - if ($showNativeToolsWarning) { - Write-Warning $errMsg + $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) + if ($ToolDirectories -eq $null) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 + $ToolDirectory = $ToolDirectories[0] + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } + } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + } } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +193,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index b7bcfe38caf..2cb42e3e7ba 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index ccd2a340934..54775f6a459 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -83,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.110.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.130.0 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index c3ccd64dd37..aaeacddce1e 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -259,7 +259,14 @@ stages: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] pool: - vmImage: 'windows-2019' + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-windows-2022 steps: - task: PowerShell@2 displayName: Publish Using Darc diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 15b2ef4cec8..a0f13b99c02 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -604,6 +604,10 @@ function InitializeNativeTools() { InstallDirectory = "$ToolsDir" } } + if ($env:NativeToolsOnMachine) { + Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." + $nativeArgs += @{ PathPromotion = $true } + } & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } diff --git a/global.json b/global.json index cf3f2800c86..75e55e50687 100644 --- a/global.json +++ b/global.json @@ -12,6 +12,6 @@ }, "msbuild-sdks": { "Microsoft.Build.CentralPackageVersions": "2.0.1", - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22276.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.22526.12" } } From 4a3a06f6542a37031f8691bd7b24d4c2c244c610 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Thu, 1 Dec 2022 11:56:34 -0600 Subject: [PATCH 2/2] Move BAR publish to windows-latest The win2016 images aren't live any more. --- .vsts-dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml index d76ea236266..22315da3e0b 100644 --- a/.vsts-dotnet.yml +++ b/.vsts-dotnet.yml @@ -231,7 +231,7 @@ stages: dependsOn: - Windows_NT pool: - vmImage: vs2017-win2016 + vmImage: windows-latest - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: eng\common\templates\post-build\post-build.yml