diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index a1b1333723e..9120b2d2129 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -54,7 +54,6 @@ https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json Build configuration: 'Debug' or 'Release' (short: -c)" - Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" @@ -79,7 +77,6 @@ function Build { InitializeCustomToolset $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } - $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -91,7 +88,6 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` - $platformArg ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` @@ -133,8 +129,9 @@ try { Build } catch { + Write-Host $_ + Write-Host $_.Exception Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/build.sh b/eng/common/build.sh index 6236fc4d38c..ce846d888df 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -66,7 +66,6 @@ ci=false warn_as_error=true node_reuse=true binary_log=false -pipelines_log=false projects='' configuration='Debug' @@ -93,9 +92,6 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; - -pipelineslog|-pl) - pipelines_log=true - ;; -restore|-r) restore=true ;; @@ -150,7 +146,6 @@ while [[ $# > 0 ]]; do done if [[ "$ci" == true ]]; then - pipelines_log=true binary_log=true node_reuse=false fi diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index ed70e0a86eb..ba16e991c7e 100644 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -157,15 +157,15 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic +fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel fetch_tizen_pkgs noarch linux-glibc-devel Inform "fetch coreclr packages" -fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel tizen-release lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel Inform "Initialize standard unified" fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release +fetch_tizen_pkgs armv7l gssdp gssdp-devel diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 7c4e122651c..34d3d2ba1fe 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -181,7 +181,7 @@ if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then fi if [ -z "$__RootfsDir" ]; then - __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" + __RootfsDir="$__CrossDir/rootfs/$__BuildArch" fi if [ -d "$__RootfsDir" ]; then diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 8854d979f37..dea7cdd903b 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -11,10 +11,10 @@ function InstallDarcCli ($darcVersion) { $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list -g + $toolList = Invoke-Expression "& `"$dotnet`" tool list -g" if ($toolList -like "*$darcCliPackageName*") { - & "$dotnet" tool uninstall $darcCliPackageName -g + Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g" } # If the user didn't explicitly specify the darc version, @@ -22,12 +22,12 @@ function InstallDarcCli ($darcVersion) { if (-not $darcVersion) { $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" } InstallDarcCli $darcVersion diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 0b629b8301a..5987943fd6f 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -8,14 +8,9 @@ Param( . $PSScriptRoot\tools.ps1 -$dotnetRoot = Join-Path $RepoRoot ".dotnet" - -$installdir = $dotnetRoot try { - if ($architecture -and $architecture.Trim() -eq "x86") { - $installdir = Join-Path $installdir "x86" - } - InstallDotNet $installdir $version $architecture $runtime $true + $dotnetRoot = Join-Path $RepoRoot ".dotnet" + InstallDotNet $dotnetRoot $version $architecture $runtime $true } catch { Write-Host $_ @@ -24,4 +19,4 @@ catch { ExitWithExitCode 1 } -ExitWithExitCode 0 +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index b056e4c1ac2..a05b84f7987 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -25,7 +25,7 @@ function CheckExitCode ([string]$stage) try { Push-Location $PSScriptRoot - + Write-Host "Installing darc..." . .\darc-init.ps1 -darcVersion $darcVersion CheckExitCode "Running darc-init" @@ -40,9 +40,9 @@ try { $darcExe = "$env:USERPROFILE\.dotnet\tools" $darcExe = Resolve-Path "$darcExe\darc.exe" - + Create-Directory $outputFolder - + # Generate 3 graph descriptions: # 1. Flat with coherency information # 2. Graphviz (dot) file @@ -51,26 +51,26 @@ try { $graphVizImageFilePath = "$outputFolder\graph.png" $normalGraphFilePath = "$outputFolder\graph-full.txt" $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) - + $baseOptions = "get-dependency-graph --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken" + if ($includeToolset) { Write-Host "Toolsets will be included in the graph..." - $baseOptions += @( "--include-toolset" ) + $baseOptions += " --include-toolset" } Write-Host "Generating standard dependency graph..." - & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath + Invoke-Expression "& `"$darcExe`" $baseOptions --output-file $normalGraphFilePath" CheckExitCode "Generating normal dependency graph" Write-Host "Generating flat dependency graph and graphviz file..." - & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath + Invoke-Expression "& `"$darcExe`" $baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath" CheckExitCode "Generating flat and graphviz dependency graph" Write-Host "Generating graph image $graphVizFilePath" $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" + Invoke-Expression "& `"$dotFilePath`" -Tpng -o'$graphVizImageFilePath' `"$graphVizFilePath`"" CheckExitCode "Generating graphviz image" - + Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" } catch { diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index 9d18645f455..a4306bd37e1 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -79,27 +79,28 @@ try { $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } + $LocalInstallerCommand = $InstallerPath + $LocalInstallerCommand += " -ToolName $ToolName" + $LocalInstallerCommand += " -InstallPath $InstallBin" + $LocalInstallerCommand += " -BaseUri $BaseUri" + $LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir" + $LocalInstallerCommand += " -Version $ToolVersion" if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } + $LocalInstallerCommand += " -Verbose" } if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { if($Force) { - $LocalInstallerArguments += @{ Force = $True } + $LocalInstallerCommand += " -Force" } } if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } + $LocalInstallerCommand += " -Clean" } Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" - & $InstallerPath @LocalInstallerArguments + Write-Verbose "Executing '$LocalInstallerCommand'" + Invoke-Expression "$LocalInstallerCommand" if ($LASTEXITCODE -Ne "0") { $errMsg = "$ToolName installation failed" if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 7a34c7e8a42..f286ae0cde2 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -209,7 +209,7 @@ function New-ScriptShim { Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") } - & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory + Invoke-Expression "$ShimDirectory\WinShimmer\winshimmer.exe $ShimName $ToolFilePath $ShimDirectory" return $True } catch { diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 deleted file mode 100644 index 7b61376f8aa..00000000000 --- a/eng/common/pipeline-logging-functions.ps1 +++ /dev/null @@ -1,233 +0,0 @@ -# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. - -# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 - -$script:loggingCommandPrefix = '##vso[' -$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? - New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } - New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } - New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } - New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } -) -# TODO: BUG: Escape % ??? -# TODO: Add test to verify don't need to escape "=". - -function Write-PipelineTelemetryError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Category, - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $PSBoundParameters.Remove("Category") | Out-Null - - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - $PSBoundParameters.Remove("Message") | Out-Null - $PSBoundParameters.Add("Message", $Message) - - Write-PipelineTaskError @PSBoundParameters -} - -function Write-PipelineTaskError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - if(!$ci) { - if($Type -eq 'error') { - Write-Host $Message -ForegroundColor Red - return - } - elseif ($Type -eq 'warning') { - Write-Host $Message -ForegroundColor Yellow - return - } - } - - if(($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return - } - if(-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters - } - - function Write-PipelineSetVariable { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput) - - if($ci) { - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = 'true' - } -AsOutput:$AsOutput - } - } - - function Write-PipelinePrependPath { - [CmdletBinding()] - param( - [Parameter(Mandatory=$true)] - [string]$Path, - [switch]$AsOutput) - if($ci) { - Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } - } - -<######################################## -# Private functions. -########################################> -function Format-LoggingCommandData { - [CmdletBinding()] - param([string]$Value, [switch]$Reverse) - - if (!$Value) { - return '' - } - - if (!$Reverse) { - foreach ($mapping in $script:loggingCommandEscapeMappings) { - $Value = $Value.Replace($mapping.Token, $mapping.Replacement) - } - } else { - for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { - $mapping = $script:loggingCommandEscapeMappings[$i] - $Value = $Value.Replace($mapping.Replacement, $mapping.Token) - } - } - - return $Value -} - -function Format-LoggingCommand { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Area, - [Parameter(Mandatory = $true)] - [string]$Event, - [string]$Data, - [hashtable]$Properties) - - # Append the preamble. - [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder - $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) - - # Append the properties. - if ($Properties) { - $first = $true - foreach ($key in $Properties.Keys) { - [string]$value = Format-LoggingCommandData $Properties[$key] - if ($value) { - if ($first) { - $null = $sb.Append(' ') - $first = $false - } else { - $null = $sb.Append(';') - } - - $null = $sb.Append("$key=$value") - } - } - } - - # Append the tail and output the value. - $Data = Format-LoggingCommandData $Data - $sb.Append(']').Append($Data).ToString() -} - -function Write-LoggingCommand { - [CmdletBinding(DefaultParameterSetName = 'Parameters')] - param( - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Area, - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Event, - [Parameter(ParameterSetName = 'Parameters')] - [string]$Data, - [Parameter(ParameterSetName = 'Parameters')] - [hashtable]$Properties, - [Parameter(Mandatory = $true, ParameterSetName = 'Object')] - $Command, - [switch]$AsOutput) - - if ($PSCmdlet.ParameterSetName -eq 'Object') { - Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput - return - } - - $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties - if ($AsOutput) { - $command - } else { - Write-Host $command - } -} - -function Write-LogIssue { - [CmdletBinding()] - param( - [ValidateSet('warning', 'error')] - [Parameter(Mandatory = $true)] - [string]$Type, - [string]$Message, - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } - if ($AsOutput) { - return $command - } - - if ($Type -eq 'error') { - $foregroundColor = $host.PrivateData.ErrorForegroundColor - $backgroundColor = $host.PrivateData.ErrorBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Red - $backgroundColor = [System.ConsoleColor]::Black - } - } else { - $foregroundColor = $host.PrivateData.WarningForegroundColor - $backgroundColor = $host.PrivateData.WarningBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Yellow - $backgroundColor = [System.ConsoleColor]::Black - } - } - - Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor -} \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh deleted file mode 100644 index 6098f9a5438..00000000000 --- a/eng/common/pipeline-logging-functions.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -function Write-PipelineTelemetryError { - local telemetry_category='' - local function_args=() - local message='' - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" - case "$opt" in - -category|-c) - telemetry_category=$2 - shift - ;; - -*) - function_args+=("$1 $2") - shift - ;; - *) - message=$* - ;; - esac - shift - done - - if [[ "$ci" != true ]]; then - echo "$message" >&2 - return - fi - - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") - - Write-PipelineTaskError $function_args -} - -function Write-PipelineTaskError { - if [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - - message_type="error" - sourcepath='' - linenumber='' - columnnumber='' - error_code='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" - case "$opt" in - -type|-t) - message_type=$2 - shift - ;; - -sourcepath|-s) - sourcepath=$2 - shift - ;; - -linenumber|-ln) - linenumber=$2 - shift - ;; - -columnnumber|-cn) - columnnumber=$2 - shift - ;; - -errcode|-e) - error_code=$2 - shift - ;; - *) - break - ;; - esac - - shift - done - - message="##vso[task.logissue" - - message="$message type=$message_type" - - if [ -n "$sourcepath" ]; then - message="$message;sourcepath=$sourcepath" - fi - - if [ -n "$linenumber" ]; then - message="$message;linenumber=$linenumber" - fi - - if [ -n "$columnnumber" ]; then - message="$message;columnnumber=$columnnumber" - fi - - if [ -n "$error_code" ]; then - message="$message;code=$error_code" - fi - - message="$message]$*" - echo "$message" -} - diff --git a/eng/common/post-build/dotnetsymbol-init.ps1 b/eng/common/post-build/dotnetsymbol-init.ps1 deleted file mode 100644 index e7659b98c8c..00000000000 --- a/eng/common/post-build/dotnetsymbol-init.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -param ( - $dotnetsymbolVersion = $null -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -$verbosity = "minimal" - -function Installdotnetsymbol ($dotnetsymbolVersion) { - $dotnetsymbolPackageName = "dotnet-symbol" - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." - } - else { - Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." - Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global - } -} - -Installdotnetsymbol $dotnetsymbolVersion diff --git a/eng/common/post-build/sourcelink-cli-init.ps1 b/eng/common/post-build/sourcelink-cli-init.ps1 deleted file mode 100644 index 9eaa25b3b50..00000000000 --- a/eng/common/post-build/sourcelink-cli-init.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -param ( - $sourcelinkCliVersion = $null -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -$verbosity = "minimal" - -function InstallSourcelinkCli ($sourcelinkCliVersion) { - $sourcelinkCliPackageName = "sourcelink" - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global - } -} - -InstallSourcelinkCli $sourcelinkCliVersion diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 deleted file mode 100644 index 8abd684e9e5..00000000000 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ /dev/null @@ -1,224 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @(".dll", ".exe", ".pdb") - - Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith(".resources.dll")) { - return - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - $Status = 0 - } - } - catch { - write-host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host "Passed." - } - else { - Write-PipelineTaskError "$PackagePath has broken SourceLink links." - } -} - -function ValidateSourceLinkLinks { - if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { - if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { - Write-PipelineTaskError "GHRepoName should be in the format / or -" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { - Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" - ExitWithExitCode 1 - } - - $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") - $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" - Write-Host $_ - ExitWithExitCode 1 - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - # Process each NuGet package in parallel - $Jobs = @() - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - Write-Host "Installing SourceLink CLI..." - Get-Location - . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion - CheckExitCode "Running sourcelink-cli-init" - - Measure-Command { ValidateSourceLinkLinks } -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 deleted file mode 100644 index 69456854e04..00000000000 --- a/eng/common/post-build/symbols-validation.ps1 +++ /dev/null @@ -1,186 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -Add-Type -AssemblyName System.IO.Compression.FileSystem - -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + "\" + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, ".pdb") - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - - $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" - - & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null - - if (Test-Path $PdbPath) { - return "PDB" - } - elseif (Test-Path $NGenPdb) { - return "NGen PDB" - } - elseif (Test-Path $SODbg) { - return "DBG for SO" - } - elseif (Test-Path $DylibDwarf) { - return "Dwarf for Dylib" - } - elseif (Test-Path $SymbolPath) { - return "Module" - } - else { - return $null - } -} - -function CountMissingSymbols { - param( - [string] $PackagePath # Path to a NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - # Extensions for which we'll look for symbols - $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") - - # How many files are missing symbol information - $MissingSymbols = 0 - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" - - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - - Get-ChildItem -Recurse $ExtractPath | - Where-Object {$RelevantExtensions -contains $_.Extension} | - ForEach-Object { - if ($_.FullName -Match "\\ref\\") { - Write-Host "`t Ignoring reference assembly file" $_.FullName - return - } - - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath - - Write-Host -NoNewLine "`t Checking file" $_.FullName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" - } - else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host "No symbols found on MSDL or SymWeb!" - } - else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host "No symbols found on MSDL!" - } - else { - Write-Host "No symbols found on SymWeb!" - } - } - } - } - - Pop-Location - - return $MissingSymbols -} - -function CheckSymbolsAvailable { - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $FileName = $_.Name - - # These packages from Arcade-Services include some native libraries that - # our current symbol uploader can't handle. Below is a workaround until - # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - - Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - - if ($Status -ne 0) { - Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" - ExitWithExitCode $exitCode - } - - Write-Host - } -} - -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - Write-Host "Installing dotnet symbol ..." - Get-Location - . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion - CheckExitCode "Running dotnetsymbol-init" - - CheckSymbolsAvailable -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config deleted file mode 100644 index 0c5451c1141..00000000000 --- a/eng/common/sdl/NuGet.config +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 deleted file mode 100644 index 74080f22d1e..00000000000 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ /dev/null @@ -1,97 +0,0 @@ -Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI pacakge (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName="master", # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory, # Required: the directory where source files are located - [string] $ArtifactsDirectory, # Required: the directory where build artifacts are located - [string] $DncEngAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCHNAME, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -#Replace repo names to the format of org/repo -if (!($Repository.contains('/'))) { - $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; -} -else{ - $RepoName = $Repository; -} - -if ($GuardianPackageName) { - $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) -} else { - $guardianCliLocation = $GuardianCliLocation -} - -$ValidPath = Test-Path $guardianCliLocation - -if ($ValidPath -eq $False) -{ - Write-Host "Invalid Guardian CLI Location." - exit 1 -} - -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -DncEngAccessToken $DncEngAccessToken -GuardianLoggerLevel $GuardianLoggerLevel -$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" - -if ($TsaOnboard) { - if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } else { - Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." - exit 1 - } -} - -if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -} -if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -} - -if ($UpdateBaseline) { - & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Update baseline" -} - -if ($TsaPublish) { - if ($TsaBranchName -and $BuildNumber) { - if (-not $TsaRepositoryName) { - $TsaRepositoryName = "$($Repository)-$($BranchName)" - } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } else { - Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." - exit 1 - } -} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 deleted file mode 100644 index cbf5c36a8f7..00000000000 --- a/eng/common/sdl/init-sdl.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $Repository, - [string] $BranchName="master", - [string] $WorkingDirectory, - [string] $DncEngAccessToken, - [string] $GuardianLoggerLevel="Standard" -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$DncEngAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" -$zipFile = "$WorkingDirectory/gdn.zip" - -Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory ".gdn") -Try -{ - # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead - Write-Host "Downloading gdn folder from internal config repostiory..." - Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile - if (Test-Path $gdnFolder) { - # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) - Remove-Item -Force -Recurse $gdnFolder - } - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) - Write-Host $gdnFolder -} Catch [System.Net.WebException] { - # if the folder does not exist, we'll do a guardian init and push it to the remote repository - Write-Host "Initializing Guardian..." - Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" - & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Error "Guardian init failed with exit code $LASTEXITCODE." - } - # We create the mainbaseline so it can be edited later - Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" - & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline - if ($LASTEXITCODE -ne 0) { - Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." - } - & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Initialize gdn folder" -} \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config deleted file mode 100644 index b054737df13..00000000000 --- a/eng/common/sdl/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 deleted file mode 100644 index cacaf8e9127..00000000000 --- a/eng/common/sdl/push-gdn.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -Param( - [string] $Repository, - [string] $BranchName="master", - [string] $GdnFolder, - [string] $DncEngAccessToken, - [string] $PushReason -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -# We create the temp directory where we'll store the sdl-config repository -$sdlDir = Join-Path $env:TEMP "sdl" -if (Test-Path $sdlDir) { - Remove-Item -Force -Recurse $sdlDir -} - -Write-Host "git clone https://dnceng:`$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" -git clone https://dnceng:$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir -if ($LASTEXITCODE -ne 0) { - Write-Error "Git clone failed with exit code $LASTEXITCODE." -} -# We copy the .gdn folder from our local run into the git repository so it can be committed -$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" -if (Get-Command Robocopy) { - Robocopy /S $GdnFolder $sdlRepositoryFolder -} else { - rsync -r $GdnFolder $sdlRepositoryFolder -} -# cd to the sdl-config directory so we can run git there -Push-Location $sdlDir -# git add . --> git commit --> git push -Write-Host "git add ." -git add . -if ($LASTEXITCODE -ne 0) { - Write-Error "Git add failed with exit code $LASTEXITCODE." -} -Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" -git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" -if ($LASTEXITCODE -ne 0) { - Write-Error "Git commit failed with exit code $LASTEXITCODE." -} -Write-Host "git push" -git push -if ($LASTEXITCODE -ne 0) { - Write-Error "Git push failed with exit code $LASTEXITCODE." -} - -# Return to the original directory -Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 deleted file mode 100644 index e6a86d03a21..00000000000 --- a/eng/common/sdl/run-sdl.ps1 +++ /dev/null @@ -1,65 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $TargetDirectory, - [string] $GdnFolder, - [string[]] $ToolsList, - [string] $UpdateBaseline, - [string] $GuardianLoggerLevel="Standard" -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -# We store config files in the r directory of .gdn -Write-Host $ToolsList -$gdnConfigPath = Join-Path $GdnFolder "r" -$ValidPath = Test-Path $GuardianCliLocation - -if ($ValidPath -eq $False) -{ - Write-Host "Invalid Guardian CLI Location." - exit 1 -} - -foreach ($tool in $ToolsList) { - $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" - $config = $False - Write-Host $tool - # We have to manually configure tools that run on source to look at the source directory only - if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - $config = $True - } - if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - $config = $True - } - - Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" - if ($config) { - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } else { - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } -} - diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 619ec68aa43..620bd3c62e7 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -57,33 +57,8 @@ jobs: /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:Configuration=$(_BuildConfig) - /v:detailed condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - task: powershell@2 - displayName: Create BARBuildId Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/BARBuildId.txt" -Value $(BARBuildId) - - task: powershell@2 - displayName: Create Channels Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/Channels.txt" -Value "$(DefaultChannels)" - - task: PublishBuildArtifacts@1 - displayName: Publish BAR BuildId to VSTS - inputs: - PathtoPublish: '$(Build.StagingDirectory)/BARBuildId.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - task: PublishBuildArtifacts@1 - displayName: Publish Channels to VSTS - inputs: - PathtoPublish: '$(Build.StagingDirectory)/Channels.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml deleted file mode 100644 index b332cb51732..00000000000 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ /dev/null @@ -1,146 +0,0 @@ -parameters: - enableSymbolValidation: true - -stages: -- stage: Publish - dependsOn: validate - variables: - - template: ../common-variables.yml - displayName: Developer Channel - jobs: - - template: ../setup-maestro-vars.yml - - - job: - displayName: Symbol Publishing - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - variables: - - group: DotNet-Symbol-Server-Pats - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download PDB Artifacts - inputs: - buildType: current - artifactName: PDBArtifacts - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:Configuration=Release - - - job: - displayName: Publish to Static Feed - dependsOn: setupMaestroVars - variables: - - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Asset Manifests - inputs: - buildType: current - artifactName: AssetManifests - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet - /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' - /p:OverrideAssetsWithSameName=true - /p:PassIfExistingItemIdentical=true - /p:Configuration=Release - - -- stage: PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - variables: - BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) - continueOnError: true - - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml deleted file mode 100644 index 0b9719da82b..00000000000 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ /dev/null @@ -1,92 +0,0 @@ -stages: -- stage: PVR_Publish - dependsOn: validate - variables: - - template: ../common-variables.yml - displayName: Validation Channel - jobs: - - template: ../setup-maestro-vars.yml - - - job: - displayName: Publish to Static Feed - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) - variables: - - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Asset Manifests - inputs: - buildType: current - artifactName: AssetManifests - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet - /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' - /p:OverrideAssetsWithSameName=true - /p:PassIfExistingItemIdentical=true - /p:Configuration=Release - - -- stage: PVR_PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - group: Publish-Build-Assets - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) - continueOnError: true - - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml deleted file mode 100644 index 97b48d97fec..00000000000 --- a/eng/common/templates/post-build/common-variables.yml +++ /dev/null @@ -1,9 +0,0 @@ -variables: - # .NET Core 3 Dev - PublicDevRelease_30_Channel_Id: 3 - - # .NET Tools - Validation - PublicValidationRelease_30_Channel_Id: 9 - - SourceLinkCLIVersion: 3.0.0 - SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml deleted file mode 100644 index 6b74475a6f8..00000000000 --- a/eng/common/templates/post-build/post-build.yml +++ /dev/null @@ -1,59 +0,0 @@ -parameters: - enableSourceLinkValidation: true - enableSigningValidation: true - enableSymbolValidation: true - -stages: -- stage: validate - dependsOn: build - displayName: Validate - jobs: - - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - - job: - displayName: Signing Validation - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:Configuration=Release - - - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: - - job: - displayName: SourceLink Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - -- template: \eng\common\templates\post-build\channels\public-dev-release.yml - parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} - -- template: \eng\common\templates\post-build\channels\public-validation-release.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml deleted file mode 100644 index d00317003b7..00000000000 --- a/eng/common/templates/post-build/promote-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: - ChannelId: 0 - -jobs: -- job: - displayName: Promote Build - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - name: ChannelId - value: ${{ parameters.ChannelId }} - - group: Publish-Build-Assets - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Add Build to Channel - inputs: - targetType: inline - script: | - $headers = @{ - "Accept" = "application/json" - "Authorization" = "Bearer $(MaestroAccessToken)" - } - Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 - enabled: false diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml deleted file mode 100644 index b40e0260a3a..00000000000 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,26 +0,0 @@ -jobs: -- job: setupMaestroVars - displayName: Setup Maestro Vars - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - script: | - . "$(Build.SourcesDirectory)/eng/common/tools.ps1" - - $BarId = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/BARBuildId.txt" - Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId - - $Channels = "" - Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/Channels.txt" | ForEach-Object { $Channels += "$_ ," } - Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 05df886f55f..d1ce577db5b 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,7 +5,6 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects @@ -36,7 +35,6 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} @@ -66,7 +64,6 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60741f03901..9ca177b82a3 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -35,7 +35,7 @@ # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } @@ -76,7 +76,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { $finished = $false try { - while (-not $process.WaitForExit(100)) { + while (-not $process.WaitForExit(100)) { # Non-blocking loop done to allow ctr-c interrupts } @@ -134,7 +134,7 @@ function InitializeDotNetCli([bool]$install) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" + Write-Host "Unable to find dotnet with SDK version '$dotnetSdkVersion'" -ForegroundColor Red ExitWithExitCode 1 } } @@ -147,10 +147,12 @@ function InitializeDotNetCli([bool]$install) { # It also ensures that VS msbuild will use the downloaded sdk targets. $env:PATH = "$dotnetRoot;$env:PATH" - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build - Write-PipelinePrependPath -Path $dotnetRoot - Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' - Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + if ($ci) { + # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + Write-Host "##vso[task.prependpath]$dotnetRoot" + Write-Host "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" + Write-Host "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" + } return $global:_DotNetInstallDir = $dotnetRoot } @@ -182,13 +184,13 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit & $installScript @installParameters if ($lastExitCode -ne 0) { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." + Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red ExitWithExitCode $lastExitCode } } # -# Locates Visual Studio MSBuild installation. +# Locates Visual Studio MSBuild installation. # The preference order for MSBuild to use is as follows: # # 1. MSBuild from an active VS command prompt @@ -205,17 +207,13 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } - $vsMinVersion = [Version]::new($vsMinVersionStr) + $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. if ($env:VSINSTALLDIR -ne $null) { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { - # Workaround for https://github.com/dotnet/roslyn/issues/35793 - # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ - $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) - - if ($msbuildVersion -ge $vsMinVersion) { + if ($msbuildCmd.Version -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path } @@ -254,7 +252,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { $env:VSINSTALLDIR = $vsInstallDir Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\") - + $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\" if (Test-Path $vsSdkInstallDir) { Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir @@ -289,13 +287,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # # The following properties of tools.vs are recognized: -# "version": "{major}.{minor}" +# "version": "{major}.{minor}" # Two part minimal VS version, e.g. "15.9", "16.0", etc. -# "components": ["componentId1", "componentId2", ...] +# "components": ["componentId1", "componentId2", ...] # Array of ids of workload components that must be available in the VS instance. # See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017 # -# Returns JSON describing the located VS instance (same format as returned by vswhere), +# Returns JSON describing the located VS instance (same format as returned by vswhere), # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ @@ -315,8 +313,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") - + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild") + if (Get-Member -InputObject $vsRequirements -Name "version") { $args += "-version" $args += $vsRequirements.version @@ -326,7 +324,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){ foreach ($component in $vsRequirements.components) { $args += "-requires" $args += $component - } + } } $vsInfo =& $vsWhereExe $args | ConvertFrom-Json @@ -356,7 +354,7 @@ function InitializeBuildTool() { if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." + Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red ExitWithExitCode 1 } @@ -365,13 +363,13 @@ function InitializeBuildTool() { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore } catch { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ + Write-Host $_ -ForegroundColor Red ExitWithExitCode 1 } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." + Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red ExitWithExitCode 1 } @@ -383,12 +381,12 @@ function GetDefaultMSBuildEngine() { if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { return "vs" } - + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { return "dotnet" } - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." + Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red ExitWithExitCode 1 } @@ -413,13 +411,11 @@ function GetSdkTaskProject([string]$taskName) { function InitializeNativeTools() { if (Get-Member -InputObject $GlobalJson -Name "native-tools") { - $nativeArgs= @{} + $nativeArgs="" if ($ci) { - $nativeArgs = @{ - InstallDirectory = "$ToolsDir" - } + $nativeArgs = "-InstallDirectory $ToolsDir" } - & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs + Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" } } @@ -441,7 +437,7 @@ function InitializeToolset() { } if (-not $restore) { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." + Write-Host "Toolset version $toolsetVersion has not been restored." -ForegroundColor Red ExitWithExitCode 1 } @@ -501,13 +497,11 @@ function MSBuild() { function MSBuild-Core() { if ($ci) { if (!$binaryLog) { - Write-PipelineTaskError -Message "Binary log must be enabled in CI build." - ExitWithExitCode 1 + throw "Binary log must be enabled in CI build." } if ($nodeReuse) { - Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." - ExitWithExitCode 1 + throw "Node reuse must be disabled in CI build." } } @@ -515,8 +509,8 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - if ($warnAsError) { - $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + if ($warnAsError) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" } foreach ($arg in $args) { @@ -524,29 +518,29 @@ function MSBuild-Core() { $cmdArgs += " `"$arg`"" } } - + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTaskError -Message "Build failed." + Write-Host "Build failed." -ForegroundColor Red $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($buildLog -ne $null) { - Write-Host "See log: $buildLog" -ForegroundColor DarkGray + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray } ExitWithExitCode $exitCode } } -function GetMSBuildBinaryLogCommandLineArgument($arguments) { +function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { $arg = $argument.Trim() if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { return $arg.Substring("/bl:".Length) - } - + } + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { return $arg.Substring("/binaryLogger:".Length) } @@ -556,8 +550,6 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } -. $PSScriptRoot\pipeline-logging-functions.ps1 - $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $ArtifactsDir = Join-Path $RepoRoot "artifacts" @@ -573,8 +565,11 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir -Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir -Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir -Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir -Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir -Write-PipelineSetVariable -Name 'TMP' -Value $TempDir +if ($ci) { + Write-Host "##vso[task.setvariable variable=Artifacts]$ArtifactsDir" + Write-Host "##vso[task.setvariable variable=Artifacts.Toolset]$ToolsetDir" + Write-Host "##vso[task.setvariable variable=Artifacts.Log]$LogDir" + + $env:TEMP = $TempDir + $env:TMP = $TempDir +} diff --git a/eng/common/tools.sh b/eng/common/tools.sh index f39aab57b99..df3eb8bce07 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -1,20 +1,8 @@ -#!/usr/bin/env bash - # Initialize variables if they aren't already defined. # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} -# Set to true to use the pipelines logger which will enable Azure logging output. -# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md -# This flag is meant as a temporary opt-opt for the feature while validate it across -# our consumers. It will be deleted in the future. -if [[ "$ci" == true ]]; then - pipelines_log=${pipelines_log:-true} -else - pipelines_log=${pipelines_log:-false} -fi - # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} @@ -77,7 +65,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" + echo "Error: Cannot find \"$key\" in $global_json_file" >&2 ExitWithExitCode 1 fi @@ -138,7 +126,7 @@ function InitializeDotNetCli { if [[ "$install" == true ]]; then InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" else - Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" + echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2 ExitWithExitCode 1 fi fi @@ -149,7 +137,7 @@ function InitializeDotNetCli { export PATH="$dotnet_root:$PATH" if [[ $ci == true ]]; then - # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build + # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build echo "##vso[task.prependpath]$dotnet_root" echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" @@ -191,7 +179,7 @@ function InstallDotNet { fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." + echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code } } @@ -228,7 +216,6 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="netcoreapp2.1" } function GetNuGetPackageCachePath { @@ -277,7 +264,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." + echo "Toolset version $toolsetVersion has not been restored." >&2 ExitWithExitCode 2 fi @@ -289,12 +276,12 @@ function InitializeToolset { fi echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" + echo "Invalid toolset path: $toolset_build_proj" >&2 ExitWithExitCode 3 fi @@ -317,27 +304,14 @@ function StopProcesses { } function MSBuild { - local args=$@ - if [[ "$pipelines_log" == true ]]; then - InitializeBuildTool - InitializeToolset - local toolset_dir="${_InitializeToolset%/*}" - local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" - args=( "${args[@]}" "-logger:$logger_path" ) - fi - - MSBuild-Core ${args[@]} -} - -function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true ]]; then - Write-PipelineTaskError "Binary log must be enabled in CI build." + echo "Binary log must be enabled in CI build." >&2 ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - Write-PipelineTaskError "Node reuse must be disabled in CI build." + echo "Node reuse must be disabled in CI build." >&2 ExitWithExitCode 1 fi fi @@ -351,13 +325,11 @@ function MSBuild-Core { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? - Write-PipelineTaskError "Build failed (exit code '$exit_code')." + echo "Build failed (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code } } -. "$scriptroot/pipeline-logging-functions.sh" - ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` @@ -390,4 +362,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi +fi \ No newline at end of file