From e9fded757bb0ab6b96d902cad06f600a759423c2 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 26 Jul 2019 12:46:26 +0000 Subject: [PATCH 1/2] Update dependencies from https://github.com/dotnet/arcade build 20190725.15 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19375.15 --- eng/Version.Details.xml | 4 +-- eng/common/init-tools-native.ps1 | 2 +- eng/common/native/CommonLibrary.psm1 | 33 +++++++++++++++++-- .../channels/public-dev-release.yml | 2 +- .../channels/public-validation-release.yml | 2 +- eng/common/tools.sh | 4 +-- global.json | 2 +- 7 files changed, 39 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1ed95d393..7a0a634b4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 3dfa62fddcde597959c323d17426f215384e773a + ef1c110152df0d500fffb87878a86f88d1ca5295 diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index eaa05880c..8cf18bcfe 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -98,7 +98,7 @@ try { } Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" & $InstallerPath @LocalInstallerArguments if ($LASTEXITCODE -Ne "0") { $errMsg = "$ToolName installation failed" diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 7a34c7e8a..2a08d5246 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -59,9 +59,38 @@ function DownloadAndExtract { -Verbose:$Verbose if ($UnzipStatus -Eq $False) { - Write-Error "Unzip failed" - return $False + # Retry Download one more time with Force=true + $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries 1 ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$True ` + -Verbose:$Verbose + + if ($DownloadRetryStatus -Eq $False) { + Write-Error "Last attempt of download failed as well" + return $False + } + + # Retry unzip again one more time with Force=true + $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$True ` + -Verbose:$Verbose + if ($UnzipRetryStatus -Eq $False) + { + Write-Error "Last attempt of unzip failed as well" + # Clean up partial zips and extracts + if (Test-Path $TempToolPath) { + Remove-Item $TempToolPath -Force + } + if (Test-Path $InstallDirectory) { + Remove-Item $InstallDirectory -Force -Recurse + } + return $False + } } + return $True } diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index b46b06992..4eaa6e4ff 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -77,7 +77,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicDevRelease_30_Channel_Id) - /p:ArtifactsCategory=.NetCore + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 02dae0937..8c4d8f6ef 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -48,7 +48,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) - /p:ArtifactsCategory=.NetCoreValidation + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 0deb01c48..738bb5669 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -77,7 +77,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" + Write-PipelineTelemetryError -category 'InitializeToolset' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -245,7 +245,7 @@ function InitializeNativeTools() { then local nativeArgs="" if [[ "$ci" == true ]]; then - nativeArgs="-InstallDirectory $tools_dir" + nativeArgs="--installDirectory $tools_dir" fi "$_script_dir/init-tools-native.sh" $nativeArgs fi diff --git a/global.json b/global.json index 0c30638a9..5c32ecde3 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19375.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19375.15" } } From d97081c4d6c26d783d257019ed327261e108e347 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 27 Jul 2019 12:39:13 +0000 Subject: [PATCH 2/2] Update dependencies from https://github.com/dotnet/arcade build 20190726.18 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19376.18 --- eng/Version.Details.xml | 4 +- eng/common/post-build/darc-gather-drop.ps1 | 17 ++-- eng/common/post-build/nuget-validation.ps1 | 5 +- eng/common/post-build/post-build-utils.ps1 | 90 +++++++++++++++++++ eng/common/post-build/promote-build.ps1 | 37 ++++---- eng/common/post-build/setup-maestro-vars.ps1 | 26 ++++++ .../post-build/sourcelink-validation.ps1 | 31 ++++--- eng/common/post-build/symbols-validation.ps1 | 29 +++--- .../post-build/trigger-subscriptions.ps1 | 48 ++++------ .../channels/internal-servicing.yml | 5 +- .../channels/public-dev-release.yml | 5 +- .../post-build/channels/public-release.yml | 5 +- .../channels/public-validation-release.yml | 9 +- .../templates/post-build/common-variables.yml | 34 +++++-- .../templates/post-build/darc-gather-drop.yml | 5 +- .../templates/post-build/post-build.yml | 5 +- .../templates/post-build/promote-build.yml | 5 +- .../post-build/setup-maestro-vars.yml | 21 +---- .../post-build/trigger-subscription.yml | 4 +- global.json | 2 +- 20 files changed, 246 insertions(+), 141 deletions(-) create mode 100644 eng/common/post-build/post-build-utils.ps1 create mode 100644 eng/common/post-build/setup-maestro-vars.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7a0a634b4..0893278ac 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ef1c110152df0d500fffb87878a86f88d1ca5295 + 316481e57ee5e6acbbf2401eb6778a1d3d48d25b diff --git a/eng/common/post-build/darc-gather-drop.ps1 b/eng/common/post-build/darc-gather-drop.ps1 index 9cc2f1a00..93a0bd832 100644 --- a/eng/common/post-build/darc-gather-drop.ps1 +++ b/eng/common/post-build/darc-gather-drop.ps1 @@ -1,13 +1,12 @@ param( - [Parameter(Mandatory=$true)][string] $BarBuildId, # ID of the build which assets should be downloaded - [Parameter(Mandatory=$true)][string] $MaestroAccessToken, # Token used to access Maestro API - [Parameter(Mandatory=$true)][string] $DropLocation # Where the assets should be downloaded to + [Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded + [Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", # Maestro API URL + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" # Version of Maestro API to use ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 try { Write-Host "Installing DARC ..." @@ -24,8 +23,8 @@ try { --continue-on-error ` --id $BarBuildId ` --output-dir $DropLocation ` - --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ ` - --password $MaestroAccessToken ` + --bar-uri $MaestroApiEndpoint ` + --password $MaestroApiAccessToken ` --latest-location } catch { diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 index 1bdced1e3..78ed0d540 100644 --- a/eng/common/post-build/nuget-validation.ps1 +++ b/eng/common/post-build/nuget-validation.ps1 @@ -6,10 +6,7 @@ param( [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 try { $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1 new file mode 100644 index 000000000..551ae113f --- /dev/null +++ b/eng/common/post-build/post-build-utils.ps1 @@ -0,0 +1,90 @@ +# Most of the functions in this file require the variables `MaestroApiEndPoint`, +# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available. + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +. $PSScriptRoot\..\tools.ps1 + +function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") { + Validate-MaestroVars + + $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $headers.Add('Accept', $ContentType) + $headers.Add('Authorization',"Bearer $MaestroApiAccessToken") + return $headers +} + +function Get-MaestroChannel([int]$ChannelId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders + $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion" + + $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + return $result +} + +function Get-MaestroBuild([int]$BuildId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion" + + $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + return $result +} + +function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) { + Validate-MaestroVars + + $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository) + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion" + + $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + return $result +} + +function Trigger-Subscription([string]$SubscriptionId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion" + Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null +} + +function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion" + Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null +} + +function Validate-MaestroVars { + try { + Get-Variable MaestroApiEndPoint -Scope Global | Out-Null + Get-Variable MaestroApiVersion -Scope Global | Out-Null + Get-Variable MaestroApiAccessToken -Scope Global | Out-Null + + if (!($MaestroApiEndPoint -Match "^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$")) { + Write-PipelineTaskError "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" + ExitWithExitCode 1 + } + + if (!($MaestroApiVersion -Match "^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) { + Write-PipelineTaskError "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" + ExitWithExitCode 1 + } + } + catch { + Write-PipelineTaskError "Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script." + Write-Host $_ + ExitWithExitCode 1 + } +} diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/promote-build.ps1 index 84a608fa5..e5ae85f25 100644 --- a/eng/common/post-build/promote-build.ps1 +++ b/eng/common/post-build/promote-build.ps1 @@ -1,30 +1,25 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $ChannelId, - [Parameter(Mandatory=$true)][string] $BarToken, - [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", - [string] $ApiVersion = "2019-01-16" + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -function Get-Headers([string]$accept, [string]$barToken) { - $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $headers.Add('Accept',$accept) - $headers.Add('Authorization',"Bearer $barToken") - return $headers -} +. $PSScriptRoot\post-build-utils.ps1 try { - $maestroHeaders = Get-Headers 'application/json' $BarToken + # Check that the channel we are going to promote the build to exist + $channelInfo = Get-MaestroChannel -ChannelId $ChannelId - # Get info about which channels the build has already been promoted to - $getBuildApiEndpoint = "$MaestroEndpoint/api/builds/${BuildId}?api-version=$ApiVersion" - $buildInfo = Invoke-WebRequest -Method Get -Uri $getBuildApiEndpoint -Headers $maestroHeaders | ConvertFrom-Json + if (!$channelInfo) { + Write-Host "Channel with BAR ID $ChannelId was not found in BAR!" + ExitWithExitCode 1 + } + # Get info about which channels the build has already been promoted to + $buildInfo = Get-MaestroBuild -BuildId $BuildId + if (!$buildInfo) { Write-Host "Build with BAR ID $BuildId was not found in BAR!" ExitWithExitCode 1 @@ -40,10 +35,10 @@ try { } } - Write-Host "Build not present in channel $ChannelId. Promoting build ... " + Write-Host "Promoting build '$BuildId' to channel '$ChannelId'." + + Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId - $promoteBuildApiEndpoint = "$maestroEndpoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$ApiVersion" - Invoke-WebRequest -Method Post -Uri $promoteBuildApiEndpoint -Headers $maestroHeaders Write-Host "done." } catch { diff --git a/eng/common/post-build/setup-maestro-vars.ps1 b/eng/common/post-build/setup-maestro-vars.ps1 new file mode 100644 index 000000000..d7f64dc63 --- /dev/null +++ b/eng/common/post-build/setup-maestro-vars.ps1 @@ -0,0 +1,26 @@ +param( + [Parameter(Mandatory=$true)][string] $ReleaseConfigsPath # Full path to ReleaseConfigs.txt asset +) + +. $PSScriptRoot\post-build-utils.ps1 + +try { + $Content = Get-Content $ReleaseConfigsPath + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 + + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 8abd684e9..41e01ae6e 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -6,10 +6,7 @@ param( [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 # Cache/HashMap (File -> Exist flag) used to consult whether a file exist # in the repository at a specific commit point. This is populated by inserting @@ -200,21 +197,27 @@ function ValidateSourceLinkLinks { } } -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode +function InstallSourcelinkCli { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global } } try { - Write-Host "Installing SourceLink CLI..." - Get-Location - . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion - CheckExitCode "Running sourcelink-cli-init" + InstallSourcelinkCli - Measure-Command { ValidateSourceLinkLinks } + ValidateSourceLinkLinks } catch { Write-Host $_ diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 69456854e..d5ec51b15 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -4,10 +4,7 @@ param( [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -162,19 +159,25 @@ function CheckSymbolsAvailable { } } -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode +function Installdotnetsymbol { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global } } try { - Write-Host "Installing dotnet symbol ..." - Get-Location - . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion - CheckExitCode "Running dotnetsymbol-init" + Installdotnetsymbol CheckSymbolsAvailable } diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index 1a91dab03..926d5b455 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -1,33 +1,20 @@ -param( +param( [Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][int] $ChannelId, - [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", - [string] $BarToken, - [string] $ApiVersion = "2019-01-16" + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -function Get-Headers([string]$accept, [string]$barToken) { - $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $headers.Add('Accept',$accept) - $headers.Add('Authorization',"Bearer $barToken") - return $headers -} +. $PSScriptRoot\post-build-utils.ps1 # Get all the $SourceRepo subscriptions $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') -$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSourceRepo&api-version=$apiVersion" -$headers = Get-Headers 'application/json' $barToken - -$subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json +$subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId if (!$subscriptions) { Write-Host "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" - return + ExitWithExitCode 0 } $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] @@ -36,21 +23,18 @@ $failedTriggeredSubscription = $false # Get all enabled subscriptions that need dependency flow on 'everyBuild' foreach ($subscription in $subscriptions) { if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { - Write-Host "$subscription.id" + Write-Host "Should trigger this subscription: $subscription.id" [void]$subscriptionsToTrigger.Add($subscription.id) } } foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { try { - $triggerSubscriptionApiEndpoint = "$maestroEndpoint/api/subscriptions/$subscriptionToTrigger/trigger?api-version=$apiVersion" - $headers = Get-Headers 'application/json' $BarToken - - Write-Host "Triggering subscription '$subscriptionToTrigger'..." + Write-Host "Triggering subscription '$subscriptionToTrigger'." - Invoke-WebRequest -Uri $triggerSubscriptionApiEndpoint -Headers $headers -Method Post + Trigger-Subscription -SubscriptionId $subscriptionToTrigger - Write-Host "Subscription '$subscriptionToTrigger' triggered!" + Write-Host "done." } catch { @@ -61,9 +45,13 @@ foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { } } -if ($failedTriggeredSubscription) { +if ($subscriptionsToTrigger.Count -eq 0) { + Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." +} +elseif ($failedTriggeredSubscription) { Write-Host "At least one subscription failed to be triggered..." ExitWithExitCode 1 } - -Write-Host "All subscriptions were triggered successfully!" +else { + Write-Host "All subscriptions were triggered successfully!" +} diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/internal-servicing.yml index 5c07b6692..12fd2b465 100644 --- a/eng/common/templates/post-build/channels/internal-servicing.yml +++ b/eng/common/templates/post-build/channels/internal-servicing.yml @@ -41,7 +41,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -87,8 +86,8 @@ stages: /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index 4eaa6e4ff..b0f085b14 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -41,7 +41,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -87,8 +86,8 @@ stages: /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/public-release.yml index a31d37139..4c63fb43f 100644 --- a/eng/common/templates/post-build/channels/public-release.yml +++ b/eng/common/templates/post-build/channels/public-release.yml @@ -41,7 +41,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -87,8 +86,8 @@ stages: /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 8c4d8f6ef..1089ac5fa 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -12,7 +12,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -48,7 +47,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) - /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) @@ -58,13 +57,13 @@ stages: /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' /p:Configuration=Release - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 42df4ae77..bd0bc5e4d 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,21 +1,39 @@ variables: + - group: Publish-Build-Assets + # .NET Core 3 Dev - PublicDevRelease_30_Channel_Id: 3 + - name: PublicDevRelease_30_Channel_Id + value: 3 # .NET Tools - Validation - PublicValidationRelease_30_Channel_Id: 9 + - name: PublicValidationRelease_30_Channel_Id + value: 9 # .NET Core 3.0 Internal Servicing - InternalServicing_30_Channel_Id: 184 + - name: InternalServicing_30_Channel_Id + value: 184 # .NET Core 3.0 Release - PublicRelease_30_Channel_Id: 19 + - name: PublicRelease_30_Channel_Id + value: 19 # Whether the build is internal or not - IsInternalBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} # Storage account name for proxy-backed feeds - ProxyBackedFeedsAccountName: dotnetfeed + - name: ProxyBackedFeedsAccountName + value: dotnetfeed + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2019-01-16" - SourceLinkCLIVersion: 3.0.0 - SymbolToolVersion: 1.0.1 + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 diff --git a/eng/common/templates/post-build/darc-gather-drop.yml b/eng/common/templates/post-build/darc-gather-drop.yml index e0a9f0a6d..f4e3bfcf5 100644 --- a/eng/common/templates/post-build/darc-gather-drop.yml +++ b/eng/common/templates/post-build/darc-gather-drop.yml @@ -7,7 +7,6 @@ jobs: dependsOn: setupMaestroVars condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) variables: - - group: Publish-Build-Assets - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] pool: @@ -19,4 +18,6 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/darc-gather-drop.ps1 arguments: -BarBuildId $(BARBuildId) -DropLocation $(Agent.BuildDirectory)/Temp/Drop/ - -MaestroAccessToken $(MaestroAccessToken) + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index daa799259..0872db4ed 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -7,9 +7,12 @@ parameters: enable: false params: '' + # Which stages should finish execution before post-build stages start + dependsOn: [build] + stages: - stage: validate - dependsOn: build + dependsOn: ${{ parameters.dependsOn }} displayName: Validate jobs: - ${{ if eq(parameters.enableNugetValidation, 'true') }}: diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml index af48b0b33..9387c583b 100644 --- a/eng/common/templates/post-build/promote-build.yml +++ b/eng/common/templates/post-build/promote-build.yml @@ -11,7 +11,6 @@ jobs: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: ChannelId value: ${{ parameters.ChannelId }} - - group: Publish-Build-Assets pool: vmImage: 'windows-2019' steps: @@ -21,4 +20,6 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 arguments: -BuildId $(BARBuildId) -ChannelId $(ChannelId) - -BarToken $(MaestroAccessToken) + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index f6120dc1e..56242b068 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -14,22 +14,5 @@ jobs: name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - script: | - # This is needed to make Write-PipelineSetVariable works in this context - $ci = $true - - . "$(Build.SourcesDirectory)/eng/common/tools.ps1" - - $Content = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - - $BarId = $Content | Select -Index 0 - - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - - $IsStableBuild = $Content | Select -Index 2 - - Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId - Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" - Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild + filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1 + arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt' diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml index 3915cdcd1..da669030d 100644 --- a/eng/common/templates/post-build/trigger-subscription.yml +++ b/eng/common/templates/post-build/trigger-subscription.yml @@ -8,4 +8,6 @@ steps: filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 arguments: -SourceRepo $(Build.Repository.Uri) -ChannelId ${{ parameters.ChannelId }} - -BarToken $(MaestroAccessToken) + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/global.json b/global.json index 5c32ecde3..35400e066 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19375.15" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19376.18" } }