From a98e85c3d12e06b7addd1be3a8d0fa471d2d87e8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 17 Jun 2019 10:23:50 -0700 Subject: [PATCH 01/11] Update dependencies from https://github.com/dotnet/arcade build 20190615.2 (#7010) - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19315.2 --- eng/Version.Details.xml | 4 +- eng/Versions.props | 2 +- eng/common/PublishToPackageFeed.proj | 1 + eng/common/build.ps1 | 7 +- eng/common/build.sh | 5 + eng/common/cross/armel/tizen-fetch.sh | 6 +- eng/common/cross/build-rootfs.sh | 2 +- eng/common/darc-init.ps1 | 8 +- eng/common/dotnet-install.ps1 | 11 +- eng/common/generate-graph-files.ps1 | 20 +- eng/common/init-tools-native.ps1 | 21 +- eng/common/native/CommonLibrary.psm1 | 2 +- eng/common/pipeline-logging-functions.ps1 | 233 ++++++++++++++++++ eng/common/pipeline-logging-functions.sh | 102 ++++++++ eng/common/post-build/dotnetsymbol-init.ps1 | 29 +++ eng/common/post-build/sourcelink-cli-init.ps1 | 29 +++ .../post-build/sourcelink-validation.ps1 | 224 +++++++++++++++++ eng/common/post-build/symbols-validation.ps1 | 186 ++++++++++++++ eng/common/sdl/NuGet.config | 13 + eng/common/sdl/execute-all-sdl-tools.ps1 | 97 ++++++++ eng/common/sdl/init-sdl.ps1 | 48 ++++ eng/common/sdl/packages.config | 4 + eng/common/sdl/push-gdn.ps1 | 51 ++++ eng/common/sdl/run-sdl.ps1 | 65 +++++ .../templates/job/publish-build-assets.yml | 25 ++ .../channels/public-dev-release.yml | 146 +++++++++++ .../channels/public-validation-release.yml | 92 +++++++ .../templates/post-build/common-variables.yml | 9 + .../templates/post-build/post-build.yml | 59 +++++ .../templates/post-build/promote-build.yml | 28 +++ .../post-build/setup-maestro-vars.yml | 26 ++ eng/common/templates/steps/send-to-helix.yml | 3 + eng/common/tools.ps1 | 101 ++++---- eng/common/tools.sh | 50 +++- global.json | 2 +- 35 files changed, 1613 insertions(+), 98 deletions(-) create mode 100644 eng/common/pipeline-logging-functions.ps1 create mode 100644 eng/common/pipeline-logging-functions.sh create mode 100644 eng/common/post-build/dotnetsymbol-init.ps1 create mode 100644 eng/common/post-build/sourcelink-cli-init.ps1 create mode 100644 eng/common/post-build/sourcelink-validation.ps1 create mode 100644 eng/common/post-build/symbols-validation.ps1 create mode 100644 eng/common/sdl/NuGet.config create mode 100644 eng/common/sdl/execute-all-sdl-tools.ps1 create mode 100644 eng/common/sdl/init-sdl.ps1 create mode 100644 eng/common/sdl/packages.config create mode 100644 eng/common/sdl/push-gdn.ps1 create mode 100644 eng/common/sdl/run-sdl.ps1 create mode 100644 eng/common/templates/post-build/channels/public-dev-release.yml create mode 100644 eng/common/templates/post-build/channels/public-validation-release.yml create mode 100644 eng/common/templates/post-build/common-variables.yml create mode 100644 eng/common/templates/post-build/post-build.yml create mode 100644 eng/common/templates/post-build/promote-build.yml create mode 100644 eng/common/templates/post-build/setup-maestro-vars.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dcd15fa1e8b..0091420aa13 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 670f6ee1a619a2a7c84cfdfe2a1c84fbe94e1c6b + aa4285be7fab64e2b6e62e4d5688ea50931c407c diff --git a/eng/Versions.props b/eng/Versions.props index e4ae8526a0b..3fc7a08c130 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -178,4 +178,4 @@ 5.22.2.1 2.0.187 - \ No newline at end of file + diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index 9120b2d2129..a1b1333723e 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -54,6 +54,7 @@ https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" @@ -77,6 +79,7 @@ function Build { InitializeCustomToolset $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -88,6 +91,7 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` + $platformArg ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` @@ -129,9 +133,8 @@ try { Build } catch { - Write-Host $_ - Write-Host $_.Exception Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/build.sh b/eng/common/build.sh index ce846d888df..6236fc4d38c 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -66,6 +66,7 @@ ci=false warn_as_error=true node_reuse=true binary_log=false +pipelines_log=false projects='' configuration='Debug' @@ -92,6 +93,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -pipelineslog|-pl) + pipelines_log=true + ;; -restore|-r) restore=true ;; @@ -146,6 +150,7 @@ while [[ $# > 0 ]]; do done if [[ "$ci" == true ]]; then + pipelines_log=true binary_log=true node_reuse=false fi diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index ba16e991c7e..ed70e0a86eb 100644 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -157,15 +157,15 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel +fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic fetch_tizen_pkgs noarch linux-glibc-devel Inform "fetch coreclr packages" -fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel tizen-release lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel Inform "Initialize standard unified" fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l gssdp gssdp-devel +fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 34d3d2ba1fe..7c4e122651c 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -181,7 +181,7 @@ if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then fi if [ -z "$__RootfsDir" ]; then - __RootfsDir="$__CrossDir/rootfs/$__BuildArch" + __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" fi if [ -d "$__RootfsDir" ]; then diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index dea7cdd903b..8854d979f37 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -11,10 +11,10 @@ function InstallDarcCli ($darcVersion) { $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = Invoke-Expression "& `"$dotnet`" tool list -g" + $toolList = & "$dotnet" tool list -g if ($toolList -like "*$darcCliPackageName*") { - Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g" + & "$dotnet" tool uninstall $darcCliPackageName -g } # If the user didn't explicitly specify the darc version, @@ -22,12 +22,12 @@ function InstallDarcCli ($darcVersion) { if (-not $darcVersion) { $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g } InstallDarcCli $darcVersion diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 5987943fd6f..0b629b8301a 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -8,9 +8,14 @@ Param( . $PSScriptRoot\tools.ps1 +$dotnetRoot = Join-Path $RepoRoot ".dotnet" + +$installdir = $dotnetRoot try { - $dotnetRoot = Join-Path $RepoRoot ".dotnet" - InstallDotNet $dotnetRoot $version $architecture $runtime $true + if ($architecture -and $architecture.Trim() -eq "x86") { + $installdir = Join-Path $installdir "x86" + } + InstallDotNet $installdir $version $architecture $runtime $true } catch { Write-Host $_ @@ -19,4 +24,4 @@ catch { ExitWithExitCode 1 } -ExitWithExitCode 0 \ No newline at end of file +ExitWithExitCode 0 diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index a05b84f7987..b056e4c1ac2 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -25,7 +25,7 @@ function CheckExitCode ([string]$stage) try { Push-Location $PSScriptRoot - + Write-Host "Installing darc..." . .\darc-init.ps1 -darcVersion $darcVersion CheckExitCode "Running darc-init" @@ -40,9 +40,9 @@ try { $darcExe = "$env:USERPROFILE\.dotnet\tools" $darcExe = Resolve-Path "$darcExe\darc.exe" - + Create-Directory $outputFolder - + # Generate 3 graph descriptions: # 1. Flat with coherency information # 2. Graphviz (dot) file @@ -51,26 +51,26 @@ try { $graphVizImageFilePath = "$outputFolder\graph.png" $normalGraphFilePath = "$outputFolder\graph-full.txt" $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = "get-dependency-graph --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken" - + $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) + if ($includeToolset) { Write-Host "Toolsets will be included in the graph..." - $baseOptions += " --include-toolset" + $baseOptions += @( "--include-toolset" ) } Write-Host "Generating standard dependency graph..." - Invoke-Expression "& `"$darcExe`" $baseOptions --output-file $normalGraphFilePath" + & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath CheckExitCode "Generating normal dependency graph" Write-Host "Generating flat dependency graph and graphviz file..." - Invoke-Expression "& `"$darcExe`" $baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath" + & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath CheckExitCode "Generating flat and graphviz dependency graph" Write-Host "Generating graph image $graphVizFilePath" $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - Invoke-Expression "& `"$dotFilePath`" -Tpng -o'$graphVizImageFilePath' `"$graphVizFilePath`"" + & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" CheckExitCode "Generating graphviz image" - + Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" } catch { diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index a4306bd37e1..9d18645f455 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -79,28 +79,27 @@ try { $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value - $LocalInstallerCommand = $InstallerPath - $LocalInstallerCommand += " -ToolName $ToolName" - $LocalInstallerCommand += " -InstallPath $InstallBin" - $LocalInstallerCommand += " -BaseUri $BaseUri" - $LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir" - $LocalInstallerCommand += " -Version $ToolVersion" + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } if ($Verbose) { - $LocalInstallerCommand += " -Verbose" + $LocalInstallerArguments += @{ Verbose = $True } } if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { if($Force) { - $LocalInstallerCommand += " -Force" + $LocalInstallerArguments += @{ Force = $True } } } if ($Clean) { - $LocalInstallerCommand += " -Clean" + $LocalInstallerArguments += @{ Clean = $True } } Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$LocalInstallerCommand'" - Invoke-Expression "$LocalInstallerCommand" + Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" + & $InstallerPath @LocalInstallerArguments if ($LASTEXITCODE -Ne "0") { $errMsg = "$ToolName installation failed" if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index f286ae0cde2..7a34c7e8a42 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -209,7 +209,7 @@ function New-ScriptShim { Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") } - Invoke-Expression "$ShimDirectory\WinShimmer\winshimmer.exe $ShimName $ToolFilePath $ShimDirectory" + & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory return $True } catch { diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 new file mode 100644 index 00000000000..7b61376f8aa --- /dev/null +++ b/eng/common/pipeline-logging-functions.ps1 @@ -0,0 +1,233 @@ +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. + +# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 + +$script:loggingCommandPrefix = '##vso[' +$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? + New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } + New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } + New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } + New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } +) +# TODO: BUG: Escape % ??? +# TODO: Add test to verify don't need to escape "=". + +function Write-PipelineTelemetryError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Category, + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $PSBoundParameters.Remove("Category") | Out-Null + + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + $PSBoundParameters.Remove("Message") | Out-Null + $PSBoundParameters.Add("Message", $Message) + + Write-PipelineTaskError @PSBoundParameters +} + +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + if(!$ci) { + if($Type -eq 'error') { + Write-Host $Message -ForegroundColor Red + return + } + elseif ($Type -eq 'warning') { + Write-Host $Message -ForegroundColor Yellow + return + } + } + + if(($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + if(-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters + } + + function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput) + + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = 'true' + } -AsOutput:$AsOutput + } + } + + function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string]$Path, + [switch]$AsOutput) + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } + } + +<######################################## +# Private functions. +########################################> +function Format-LoggingCommandData { + [CmdletBinding()] + param([string]$Value, [switch]$Reverse) + + if (!$Value) { + return '' + } + + if (!$Reverse) { + foreach ($mapping in $script:loggingCommandEscapeMappings) { + $Value = $Value.Replace($mapping.Token, $mapping.Replacement) + } + } else { + for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { + $mapping = $script:loggingCommandEscapeMappings[$i] + $Value = $Value.Replace($mapping.Replacement, $mapping.Token) + } + } + + return $Value +} + +function Format-LoggingCommand { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Area, + [Parameter(Mandatory = $true)] + [string]$Event, + [string]$Data, + [hashtable]$Properties) + + # Append the preamble. + [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder + $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) + + # Append the properties. + if ($Properties) { + $first = $true + foreach ($key in $Properties.Keys) { + [string]$value = Format-LoggingCommandData $Properties[$key] + if ($value) { + if ($first) { + $null = $sb.Append(' ') + $first = $false + } else { + $null = $sb.Append(';') + } + + $null = $sb.Append("$key=$value") + } + } + } + + # Append the tail and output the value. + $Data = Format-LoggingCommandData $Data + $sb.Append(']').Append($Data).ToString() +} + +function Write-LoggingCommand { + [CmdletBinding(DefaultParameterSetName = 'Parameters')] + param( + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Area, + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Event, + [Parameter(ParameterSetName = 'Parameters')] + [string]$Data, + [Parameter(ParameterSetName = 'Parameters')] + [hashtable]$Properties, + [Parameter(Mandatory = $true, ParameterSetName = 'Object')] + $Command, + [switch]$AsOutput) + + if ($PSCmdlet.ParameterSetName -eq 'Object') { + Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput + return + } + + $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties + if ($AsOutput) { + $command + } else { + Write-Host $command + } +} + +function Write-LogIssue { + [CmdletBinding()] + param( + [ValidateSet('warning', 'error')] + [Parameter(Mandatory = $true)] + [string]$Type, + [string]$Message, + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } + if ($AsOutput) { + return $command + } + + if ($Type -eq 'error') { + $foregroundColor = $host.PrivateData.ErrorForegroundColor + $backgroundColor = $host.PrivateData.ErrorBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Red + $backgroundColor = [System.ConsoleColor]::Black + } + } else { + $foregroundColor = $host.PrivateData.WarningForegroundColor + $backgroundColor = $host.PrivateData.WarningBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Yellow + $backgroundColor = [System.ConsoleColor]::Black + } + } + + Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor +} \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh new file mode 100644 index 00000000000..6098f9a5438 --- /dev/null +++ b/eng/common/pipeline-logging-functions.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +function Write-PipelineTelemetryError { + local telemetry_category='' + local function_args=() + local message='' + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -category|-c) + telemetry_category=$2 + shift + ;; + -*) + function_args+=("$1 $2") + shift + ;; + *) + message=$* + ;; + esac + shift + done + + if [[ "$ci" != true ]]; then + echo "$message" >&2 + return + fi + + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + + Write-PipelineTaskError $function_args +} + +function Write-PipelineTaskError { + if [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + message_type="error" + sourcepath='' + linenumber='' + columnnumber='' + error_code='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-ln) + linenumber=$2 + shift + ;; + -columnnumber|-cn) + columnnumber=$2 + shift + ;; + -errcode|-e) + error_code=$2 + shift + ;; + *) + break + ;; + esac + + shift + done + + message="##vso[task.logissue" + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + echo "$message" +} + diff --git a/eng/common/post-build/dotnetsymbol-init.ps1 b/eng/common/post-build/dotnetsymbol-init.ps1 new file mode 100644 index 00000000000..e7659b98c8c --- /dev/null +++ b/eng/common/post-build/dotnetsymbol-init.ps1 @@ -0,0 +1,29 @@ +param ( + $dotnetsymbolVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function Installdotnetsymbol ($dotnetsymbolVersion) { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global + } +} + +Installdotnetsymbol $dotnetsymbolVersion diff --git a/eng/common/post-build/sourcelink-cli-init.ps1 b/eng/common/post-build/sourcelink-cli-init.ps1 new file mode 100644 index 00000000000..9eaa25b3b50 --- /dev/null +++ b/eng/common/post-build/sourcelink-cli-init.ps1 @@ -0,0 +1,29 @@ +param ( + $sourcelinkCliVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function InstallSourcelinkCli ($sourcelinkCliVersion) { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global + } +} + +InstallSourcelinkCli $sourcelinkCliVersion diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 new file mode 100644 index 00000000000..8abd684e9e5 --- /dev/null +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -0,0 +1,224 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + . $using:PSScriptRoot\..\tools.ps1 + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" + $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" + $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" + + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + write-host $_ + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + } + catch { + + } + finally { + $zip.Dispose() + } + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } + else { + Write-PipelineTaskError "$PackagePath has broken SourceLink links." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { + Write-PipelineTaskError "GHRepoName should be in the format / or -" + ExitWithExitCode 1 + } + else { + $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; + } + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" + ExitWithExitCode 1 + } + + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + Write-Host $_ + ExitWithExitCode 1 + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing SourceLink CLI..." + Get-Location + . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion + CheckExitCode "Running sourcelink-cli-init" + + Measure-Command { ValidateSourceLinkLinks } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 new file mode 100644 index 00000000000..69456854e04 --- /dev/null +++ b/eng/common/post-build/symbols-validation.ps1 @@ -0,0 +1,186 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + + & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" + ExitWithExitCode $exitCode + } + + Write-Host + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing dotnet symbol ..." + Get-Location + . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion + CheckExitCode "Running dotnetsymbol-init" + + CheckSymbolsAvailable +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config new file mode 100644 index 00000000000..0c5451c1141 --- /dev/null +++ b/eng/common/sdl/NuGet.config @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 new file mode 100644 index 00000000000..74080f22d1e --- /dev/null +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -0,0 +1,97 @@ +Param( + [string] $GuardianPackageName, # Required: the name of guardian CLI pacakge (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName="master", # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory, # Required: the directory where source files are located + [string] $ArtifactsDirectory, # Required: the directory where build artifacts are located + [string] $DncEngAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCHNAME, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +#Replace repo names to the format of org/repo +if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; +} +else{ + $RepoName = $Repository; +} + +if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) +} else { + $guardianCliLocation = $GuardianCliLocation +} + +$ValidPath = Test-Path $guardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -DncEngAccessToken $DncEngAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" + +if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." + exit 1 + } +} + +if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} +if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} + +if ($UpdateBaseline) { + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Update baseline" +} + +if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." + exit 1 + } +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 new file mode 100644 index 00000000000..cbf5c36a8f7 --- /dev/null +++ b/eng/common/sdl/init-sdl.ps1 @@ -0,0 +1,48 @@ +Param( + [string] $GuardianCliLocation, + [string] $Repository, + [string] $BranchName="master", + [string] $WorkingDirectory, + [string] $DncEngAccessToken, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$DncEngAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$zipFile = "$WorkingDirectory/gdn.zip" + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory ".gdn") +Try +{ + # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead + Write-Host "Downloading gdn folder from internal config repostiory..." + Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile + if (Test-Path $gdnFolder) { + # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) + Remove-Item -Force -Recurse $gdnFolder + } + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) + Write-Host $gdnFolder +} Catch [System.Net.WebException] { + # if the folder does not exist, we'll do a guardian init and push it to the remote repository + Write-Host "Initializing Guardian..." + Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" + & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian init failed with exit code $LASTEXITCODE." + } + # We create the mainbaseline so it can be edited later + Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" + & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." + } + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Initialize gdn folder" +} \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config new file mode 100644 index 00000000000..b054737df13 --- /dev/null +++ b/eng/common/sdl/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 new file mode 100644 index 00000000000..cacaf8e9127 --- /dev/null +++ b/eng/common/sdl/push-gdn.ps1 @@ -0,0 +1,51 @@ +Param( + [string] $Repository, + [string] $BranchName="master", + [string] $GdnFolder, + [string] $DncEngAccessToken, + [string] $PushReason +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We create the temp directory where we'll store the sdl-config repository +$sdlDir = Join-Path $env:TEMP "sdl" +if (Test-Path $sdlDir) { + Remove-Item -Force -Recurse $sdlDir +} + +Write-Host "git clone https://dnceng:`$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +if ($LASTEXITCODE -ne 0) { + Write-Error "Git clone failed with exit code $LASTEXITCODE." +} +# We copy the .gdn folder from our local run into the git repository so it can be committed +$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" +if (Get-Command Robocopy) { + Robocopy /S $GdnFolder $sdlRepositoryFolder +} else { + rsync -r $GdnFolder $sdlRepositoryFolder +} +# cd to the sdl-config directory so we can run git there +Push-Location $sdlDir +# git add . --> git commit --> git push +Write-Host "git add ." +git add . +if ($LASTEXITCODE -ne 0) { + Write-Error "Git add failed with exit code $LASTEXITCODE." +} +Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" +git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." +} +Write-Host "git push" +git push +if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." +} + +# Return to the original directory +Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 new file mode 100644 index 00000000000..e6a86d03a21 --- /dev/null +++ b/eng/common/sdl/run-sdl.ps1 @@ -0,0 +1,65 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + [string[]] $ToolsList, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We store config files in the r directory of .gdn +Write-Host $ToolsList +$gdnConfigPath = Join-Path $GdnFolder "r" +$ValidPath = Test-Path $GuardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +foreach ($tool in $ToolsList) { + $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" + $config = $False + Write-Host $tool + # We have to manually configure tools that run on source to look at the source directory only + if ($tool -eq "credscan") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + if ($tool -eq "policheck") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + + Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" + if ($config) { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } +} + diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 620bd3c62e7..619ec68aa43 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -57,8 +57,33 @@ jobs: /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:Configuration=$(_BuildConfig) + /v:detailed condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 + displayName: Create BARBuildId Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/BARBuildId.txt" -Value $(BARBuildId) + - task: powershell@2 + displayName: Create Channels Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/Channels.txt" -Value "$(DefaultChannels)" + - task: PublishBuildArtifacts@1 + displayName: Publish BAR BuildId to VSTS + inputs: + PathtoPublish: '$(Build.StagingDirectory)/BARBuildId.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - task: PublishBuildArtifacts@1 + displayName: Publish Channels to VSTS + inputs: + PathtoPublish: '$(Build.StagingDirectory)/Channels.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml new file mode 100644 index 00000000000..b332cb51732 --- /dev/null +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -0,0 +1,146 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Developer Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + buildType: current + artifactName: PDBArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: + displayName: Publish to Static Feed + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet + /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' + /p:OverrideAssetsWithSameName=true + /p:PassIfExistingItemIdentical=true + /p:Configuration=Release + + +- stage: PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) + continueOnError: true + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml new file mode 100644 index 00000000000..0b9719da82b --- /dev/null +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -0,0 +1,92 @@ +stages: +- stage: PVR_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Validation Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Publish to Static Feed + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet + /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' + /p:OverrideAssetsWithSameName=true + /p:PassIfExistingItemIdentical=true + /p:Configuration=Release + + +- stage: PVR_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) + continueOnError: true + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml new file mode 100644 index 00000000000..97b48d97fec --- /dev/null +++ b/eng/common/templates/post-build/common-variables.yml @@ -0,0 +1,9 @@ +variables: + # .NET Core 3 Dev + PublicDevRelease_30_Channel_Id: 3 + + # .NET Tools - Validation + PublicValidationRelease_30_Channel_Id: 9 + + SourceLinkCLIVersion: 3.0.0 + SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml new file mode 100644 index 00000000000..6b74475a6f8 --- /dev/null +++ b/eng/common/templates/post-build/post-build.yml @@ -0,0 +1,59 @@ +parameters: + enableSourceLinkValidation: true + enableSigningValidation: true + enableSymbolValidation: true + +stages: +- stage: validate + dependsOn: build + displayName: Validate + jobs: + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: + - job: + displayName: Signing Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:Configuration=Release + + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: SourceLink Validation + variables: + - template: common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + +- template: \eng\common\templates\post-build\channels\public-dev-release.yml + parameters: + enableSymbolValidation: ${{ parameters.enableSymbolValidation }} + +- template: \eng\common\templates\post-build\channels\public-validation-release.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml new file mode 100644 index 00000000000..d00317003b7 --- /dev/null +++ b/eng/common/templates/post-build/promote-build.yml @@ -0,0 +1,28 @@ +parameters: + ChannelId: 0 + +jobs: +- job: + displayName: Promote Build + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: ChannelId + value: ${{ parameters.ChannelId }} + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Add Build to Channel + inputs: + targetType: inline + script: | + $headers = @{ + "Accept" = "application/json" + "Authorization" = "Bearer $(MaestroAccessToken)" + } + Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 + enabled: false diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml new file mode 100644 index 00000000000..b40e0260a3a --- /dev/null +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,26 @@ +jobs: +- job: setupMaestroVars + displayName: Setup Maestro Vars + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + script: | + . "$(Build.SourcesDirectory)/eng/common/tools.ps1" + + $BarId = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/BARBuildId.txt" + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + + $Channels = "" + Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/Channels.txt" | ForEach-Object { $Channels += "$_ ," } + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index d1ce577db5b..05df886f55f 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,6 +5,7 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects @@ -35,6 +36,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} @@ -64,6 +66,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9ca177b82a3..60741f03901 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -35,7 +35,7 @@ # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } @@ -76,7 +76,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { $finished = $false try { - while (-not $process.WaitForExit(100)) { + while (-not $process.WaitForExit(100)) { # Non-blocking loop done to allow ctr-c interrupts } @@ -134,7 +134,7 @@ function InitializeDotNetCli([bool]$install) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { - Write-Host "Unable to find dotnet with SDK version '$dotnetSdkVersion'" -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" ExitWithExitCode 1 } } @@ -147,12 +147,10 @@ function InitializeDotNetCli([bool]$install) { # It also ensures that VS msbuild will use the downloaded sdk targets. $env:PATH = "$dotnetRoot;$env:PATH" - if ($ci) { - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build - Write-Host "##vso[task.prependpath]$dotnetRoot" - Write-Host "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" - Write-Host "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" - } + # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + Write-PipelinePrependPath -Path $dotnetRoot + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -184,13 +182,13 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit & $installScript @installParameters if ($lastExitCode -ne 0) { - Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." ExitWithExitCode $lastExitCode } } # -# Locates Visual Studio MSBuild installation. +# Locates Visual Studio MSBuild installation. # The preference order for MSBuild to use is as follows: # # 1. MSBuild from an active VS command prompt @@ -207,13 +205,17 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } - $vsMinVersion = [Version]::new($vsMinVersionStr) + $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. if ($env:VSINSTALLDIR -ne $null) { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { - if ($msbuildCmd.Version -ge $vsMinVersion) { + # Workaround for https://github.com/dotnet/roslyn/issues/35793 + # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) + + if ($msbuildVersion -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path } @@ -252,7 +254,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { $env:VSINSTALLDIR = $vsInstallDir Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\") - + $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\" if (Test-Path $vsSdkInstallDir) { Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir @@ -287,13 +289,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # # The following properties of tools.vs are recognized: -# "version": "{major}.{minor}" +# "version": "{major}.{minor}" # Two part minimal VS version, e.g. "15.9", "16.0", etc. -# "components": ["componentId1", "componentId2", ...] +# "components": ["componentId1", "componentId2", ...] # Array of ids of workload components that must be available in the VS instance. # See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017 # -# Returns JSON describing the located VS instance (same format as returned by vswhere), +# Returns JSON describing the located VS instance (same format as returned by vswhere), # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ @@ -313,8 +315,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild") - + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") + if (Get-Member -InputObject $vsRequirements -Name "version") { $args += "-version" $args += $vsRequirements.version @@ -324,7 +326,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){ foreach ($component in $vsRequirements.components) { $args += "-requires" $args += $component - } + } } $vsInfo =& $vsWhereExe $args | ConvertFrom-Json @@ -354,7 +356,7 @@ function InitializeBuildTool() { if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { - Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } @@ -363,13 +365,13 @@ function InitializeBuildTool() { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore } catch { - Write-Host $_ -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { - Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 } @@ -381,12 +383,12 @@ function GetDefaultMSBuildEngine() { if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { return "vs" } - + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { return "dotnet" } - Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." ExitWithExitCode 1 } @@ -411,11 +413,13 @@ function GetSdkTaskProject([string]$taskName) { function InitializeNativeTools() { if (Get-Member -InputObject $GlobalJson -Name "native-tools") { - $nativeArgs="" + $nativeArgs= @{} if ($ci) { - $nativeArgs = "-InstallDirectory $ToolsDir" + $nativeArgs = @{ + InstallDirectory = "$ToolsDir" + } } - Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" + & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } @@ -437,7 +441,7 @@ function InitializeToolset() { } if (-not $restore) { - Write-Host "Toolset version $toolsetVersion has not been restored." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 1 } @@ -497,11 +501,13 @@ function MSBuild() { function MSBuild-Core() { if ($ci) { if (!$binaryLog) { - throw "Binary log must be enabled in CI build." + Write-PipelineTaskError -Message "Binary log must be enabled in CI build." + ExitWithExitCode 1 } if ($nodeReuse) { - throw "Node reuse must be disabled in CI build." + Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." + ExitWithExitCode 1 } } @@ -509,8 +515,8 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - if ($warnAsError) { - $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + if ($warnAsError) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" } foreach ($arg in $args) { @@ -518,29 +524,29 @@ function MSBuild-Core() { $cmdArgs += " `"$arg`"" } } - + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-Host "Build failed." -ForegroundColor Red + Write-PipelineTaskError -Message "Build failed." $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($buildLog -ne $null) { - Write-Host "See log: $buildLog" -ForegroundColor DarkGray + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray } ExitWithExitCode $exitCode } } -function GetMSBuildBinaryLogCommandLineArgument($arguments) { +function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { $arg = $argument.Trim() if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { return $arg.Substring("/bl:".Length) - } - + } + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { return $arg.Substring("/binaryLogger:".Length) } @@ -550,6 +556,8 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +. $PSScriptRoot\pipeline-logging-functions.ps1 + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $ArtifactsDir = Join-Path $RepoRoot "artifacts" @@ -565,11 +573,8 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir -if ($ci) { - Write-Host "##vso[task.setvariable variable=Artifacts]$ArtifactsDir" - Write-Host "##vso[task.setvariable variable=Artifacts.Toolset]$ToolsetDir" - Write-Host "##vso[task.setvariable variable=Artifacts.Log]$LogDir" - - $env:TEMP = $TempDir - $env:TMP = $TempDir -} +Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir +Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir +Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir +Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir +Write-PipelineSetVariable -Name 'TMP' -Value $TempDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index df3eb8bce07..f39aab57b99 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -1,8 +1,20 @@ +#!/usr/bin/env bash + # Initialize variables if they aren't already defined. # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +if [[ "$ci" == true ]]; then + pipelines_log=${pipelines_log:-true} +else + pipelines_log=${pipelines_log:-false} +fi + # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} @@ -65,7 +77,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - echo "Error: Cannot find \"$key\" in $global_json_file" >&2 + Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -126,7 +138,7 @@ function InitializeDotNetCli { if [[ "$install" == true ]]; then InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" else - echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" ExitWithExitCode 1 fi fi @@ -137,7 +149,7 @@ function InitializeDotNetCli { export PATH="$dotnet_root:$PATH" if [[ $ci == true ]]; then - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build echo "##vso[task.prependpath]$dotnet_root" echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" @@ -179,7 +191,7 @@ function InstallDotNet { fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -216,6 +228,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" + _InitializeBuildToolFramework="netcoreapp2.1" } function GetNuGetPackageCachePath { @@ -264,7 +277,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - echo "Toolset version $toolsetVersion has not been restored." >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." ExitWithExitCode 2 fi @@ -276,12 +289,12 @@ function InitializeToolset { fi echo '' > "$proj" - MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - echo "Invalid toolset path: $toolset_build_proj" >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -304,14 +317,27 @@ function StopProcesses { } function MSBuild { + local args=$@ + if [[ "$pipelines_log" == true ]]; then + InitializeBuildTool + InitializeToolset + local toolset_dir="${_InitializeToolset%/*}" + local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + args=( "${args[@]}" "-logger:$logger_path" ) + fi + + MSBuild-Core ${args[@]} +} + +function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true ]]; then - echo "Binary log must be enabled in CI build." >&2 + Write-PipelineTaskError "Binary log must be enabled in CI build." ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - echo "Node reuse must be disabled in CI build." >&2 + Write-PipelineTaskError "Node reuse must be disabled in CI build." ExitWithExitCode 1 fi fi @@ -325,11 +351,13 @@ function MSBuild { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? - echo "Build failed (exit code '$exit_code')." >&2 + Write-PipelineTaskError "Build failed (exit code '$exit_code')." ExitWithExitCode $exit_code } } +. "$scriptroot/pipeline-logging-functions.sh" + ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` @@ -362,4 +390,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi \ No newline at end of file +fi diff --git a/global.json b/global.json index 6d63a523a7d..ace8d4fc826 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19264.13", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19315.2", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From 41fbea777c9b1199cfb165034b273c025a8101ad Mon Sep 17 00:00:00 2001 From: Phillip Carter Date: Mon, 17 Jun 2019 12:06:55 -0700 Subject: [PATCH 02/11] Remove IVT to FSharp.Compiler.Server (#6997) --- .../FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj index 2a4a2218708..2c005c363fa 100644 --- a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj +++ b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj @@ -46,7 +46,6 @@ - From 30d3f7eaaf3dae776bf1efd8360526d0930fc80e Mon Sep 17 00:00:00 2001 From: Phillip Carter Date: Mon, 17 Jun 2019 12:10:31 -0700 Subject: [PATCH 03/11] No IVTs to legacy project system or property pages (#6999) --- .../FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj index 2c005c363fa..9b24fac60de 100644 --- a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj +++ b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj @@ -42,9 +42,6 @@ - - - From 395a1f472575b0a54ffd6b822201c930d4101b65 Mon Sep 17 00:00:00 2001 From: Eugene Auduchinok Date: Mon, 17 Jun 2019 22:11:13 +0300 Subject: [PATCH 04/11] PatternMatchCompilation cleanup (#6993) * PatternMatchCompilation cleanup * More cleanup --- src/fsharp/PatternMatchCompilation.fs | 220 ++++++++++++------------- src/fsharp/PatternMatchCompilation.fsi | 22 +-- 2 files changed, 119 insertions(+), 123 deletions(-) diff --git a/src/fsharp/PatternMatchCompilation.fs b/src/fsharp/PatternMatchCompilation.fs index ddd8fc129c7..0bf4df1c256 100644 --- a/src/fsharp/PatternMatchCompilation.fs +++ b/src/fsharp/PatternMatchCompilation.fs @@ -30,7 +30,6 @@ type ActionOnFailure = | FailFilter [] -/// Represents type-checked patterns type Pattern = | TPat_const of Const * range | TPat_wild of range (* note = TPat_disjs([], m), but we haven't yet removed that duplication *) @@ -720,7 +719,9 @@ let rec erasePartialPatterns inpp = | TPat_range _ | TPat_null _ | TPat_isinst _ -> inpp -and erasePartials inps = List.map erasePartialPatterns inps + +and erasePartials inps = + List.map erasePartialPatterns inps //--------------------------------------------------------------------------- @@ -740,118 +741,115 @@ let CompilePatternBasic (clausesL: TypedMatchClause list) inputTy resultTy = - // Add the targets to a match builder - // Note the input expression has already been evaluated and saved into a variable. - // Hence no need for a new sequence point. - let mbuilder = new MatchBuilder(NoSequencePointAtInvisibleBinding, exprm) - clausesL |> List.iteri (fun _i c -> mbuilder.AddTarget c.Target |> ignore) - - // Add the incomplete or rethrow match clause on demand, printing a - // warning if necessary (only if it is ever exercised) - let incompleteMatchClauseOnce = ref None + // Add the targets to a match builder. + // Note the input expression has already been evaluated and saved into a variable, + // hence no need for a new sequence point. + let matchBuilder = MatchBuilder (NoSequencePointAtInvisibleBinding, exprm) + clausesL |> List.iter (fun c -> matchBuilder.AddTarget c.Target |> ignore) + + // Add the incomplete or rethrow match clause on demand, + // printing a warning if necessary (only if it is ever exercised). + let mutable incompleteMatchClauseOnce = None let getIncompleteMatchClause refuted = - // This is lazy because emit a - // warning when the lazy thunk gets evaluated - match !incompleteMatchClauseOnce with + // This is lazy because emit a warning when the lazy thunk gets evaluated. + match incompleteMatchClauseOnce with | None -> - (* Emit the incomplete match warning *) - if warnOnIncomplete then - match actionOnFailure with - | ThrowIncompleteMatchException | IgnoreWithWarning -> - let ignoreWithWarning = (actionOnFailure = IgnoreWithWarning) - match ShowCounterExample g denv matchm refuted with - | Some(text, failingWhenClause, true) -> - warning (EnumMatchIncomplete(ignoreWithWarning, Some(text, failingWhenClause), matchm)) - | Some(text, failingWhenClause, false) -> - warning (MatchIncomplete(ignoreWithWarning, Some(text, failingWhenClause), matchm)) - | None -> - warning (MatchIncomplete(ignoreWithWarning, None, matchm)) - | _ -> - () - - let throwExpr = - match actionOnFailure with - | FailFilter -> - // Return 0 from the .NET exception filter - mkInt g matchm 0 - - | Rethrow -> - // Rethrow unmatched try-catch exn. No sequence point at the target since its not - // real code. - mkReraise matchm resultTy - - | Throw -> - // We throw instead of rethrow on unmatched try-catch in a computation expression. But why? - // Because this isn't a real .NET exception filter/handler but just a function we're passing - // to a computation expression builder to simulate one. - mkThrow matchm resultTy (exprForVal matchm origInputVal) - - | ThrowIncompleteMatchException -> - mkThrow matchm resultTy - (mkExnExpr(mk_MFCore_tcref g.fslibCcu "MatchFailureException", - [ mkString g matchm matchm.FileName - mkInt g matchm matchm.StartLine - mkInt g matchm matchm.StartColumn], matchm)) - - | IgnoreWithWarning -> - mkUnit g matchm - - // We don't emit a sequence point at any of the above cases because they don't correspond to - // user code. - // - // Note we don't emit sequence points at either the succeeding or failing - // targets of filters since if the exception is filtered successfully then we - // will run the handler and hit the sequence point there. - // That sequence point will have the pattern variables bound, which is exactly what we want. - let tg = TTarget(List.empty, throwExpr, SuppressSequencePointAtTarget ) - mbuilder.AddTarget tg |> ignore - let clause = TClause(TPat_wild matchm, None, tg, matchm) - incompleteMatchClauseOnce := Some clause - clause + // Emit the incomplete match warning. + if warnOnIncomplete then + match actionOnFailure with + | ThrowIncompleteMatchException | IgnoreWithWarning -> + let ignoreWithWarning = (actionOnFailure = IgnoreWithWarning) + match ShowCounterExample g denv matchm refuted with + | Some(text, failingWhenClause, true) -> + warning (EnumMatchIncomplete(ignoreWithWarning, Some(text, failingWhenClause), matchm)) + | Some(text, failingWhenClause, false) -> + warning (MatchIncomplete(ignoreWithWarning, Some(text, failingWhenClause), matchm)) + | None -> + warning (MatchIncomplete(ignoreWithWarning, None, matchm)) + | _ -> + () + + let throwExpr = + match actionOnFailure with + | FailFilter -> + // Return 0 from the .NET exception filter. + mkInt g matchm 0 + + | Rethrow -> + // Rethrow unmatched try-catch exn. No sequence point at the target since its not real code. + mkReraise matchm resultTy + + | Throw -> + // We throw instead of rethrow on unmatched try-catch in a computation expression. But why? + // Because this isn't a real .NET exception filter/handler but just a function we're passing + // to a computation expression builder to simulate one. + mkThrow matchm resultTy (exprForVal matchm origInputVal) + + | ThrowIncompleteMatchException -> + mkThrow matchm resultTy + (mkExnExpr(mk_MFCore_tcref g.fslibCcu "MatchFailureException", + [ mkString g matchm matchm.FileName + mkInt g matchm matchm.StartLine + mkInt g matchm matchm.StartColumn], matchm)) + + | IgnoreWithWarning -> + mkUnit g matchm + + // We don't emit a sequence point at any of the above cases because they don't correspond to user code. + // + // Note we don't emit sequence points at either the succeeding or failing targets of filters since if + // the exception is filtered successfully then we will run the handler and hit the sequence point there. + // That sequence point will have the pattern variables bound, which is exactly what we want. + let tg = TTarget(List.empty, throwExpr, SuppressSequencePointAtTarget) + let _ = matchBuilder.AddTarget tg + let clause = TClause(TPat_wild matchm, None, tg, matchm) + incompleteMatchClauseOnce <- Some clause + clause | Some c -> c - // Helpers to get the variables bound at a target. We conceptually add a dummy clause that will always succeed with a "throw" + // Helpers to get the variables bound at a target. + // We conceptually add a dummy clause that will always succeed with a "throw" let clausesA = Array.ofList clausesL - let nclauses = clausesA.Length + let nClauses = clausesA.Length let GetClause i refuted = - if i < nclauses then + if i < nClauses then clausesA.[i] - elif i = nclauses then getIncompleteMatchClause refuted + elif i = nClauses then getIncompleteMatchClause refuted else failwith "GetClause" let GetValsBoundByClause i refuted = (GetClause i refuted).BoundVals let GetWhenGuardOfClause i refuted = (GetClause i refuted).GuardExpr - // Different uses of parameterized active patterns have different identities as far as paths - // are concerned. Here we generate unique numbers that are completely different to any stamp - // by usig negative numbers. + // Different uses of parameterized active patterns have different identities as far as paths are concerned. + // Here we generate unique numbers that are completely different to any stamp by using negative numbers. let genUniquePathId() = - (newUnique()) - // Build versions of these functions which apply a dummy instantiation to the overall type arguments + // Build versions of these functions which apply a dummy instantiation to the overall type arguments. let GetSubExprOfInput, getDiscrimOfPattern = let tyargs = List.map (fun _ -> g.unit_ty) origInputValTypars let unit_tpinst = mkTyparInst origInputValTypars tyargs GetSubExprOfInput g (origInputValTypars, tyargs, unit_tpinst), getDiscrimOfPattern g unit_tpinst - // The main recursive loop of the pattern match compiler + // The main recursive loop of the pattern match compiler. let rec InvestigateFrontiers refuted frontiers = match frontiers with | [] -> failwith "CompilePattern: compile - empty clauses: at least the final clause should always succeed" - | (Frontier (i, active, valMap)) :: rest -> + | Frontier (i, active, valMap) :: rest -> - // Check to see if we've got a succeeding clause. There may still be a 'when' condition for the clause + // Check to see if we've got a succeeding clause. There may still be a 'when' condition for the clause. match active with | [] -> CompileSuccessPointAndGuard i refuted valMap rest | _ -> - (* Otherwise choose a point (i.e. a path) to investigate. *) + // Otherwise choose a point (i.e. a path) to investigate. let (Active(path, subexpr, pat)) = ChooseInvestigationPointLeftToRight frontiers match pat with // All these constructs should have been eliminated in BindProjectionPattern - | TPat_as _ | TPat_tuple _ | TPat_wild _ | TPat_disjs _ | TPat_conjs _ | TPat_recd _ -> failwith "Unexpected pattern" + | TPat_as _ | TPat_tuple _ | TPat_wild _ | TPat_disjs _ | TPat_conjs _ | TPat_recd _ -> + failwith "Unexpected pattern" - // Leaving the ones where we have real work to do + // Leaving the ones where we have real work to do. | _ -> let simulSetOfEdgeDiscrims, fallthroughPathFrontiers = ChooseSimultaneousEdges frontiers path @@ -879,7 +877,6 @@ let CompilePatternBasic finalDecisionTree and CompileSuccessPointAndGuard i refuted valMap rest = - let vs2 = GetValsBoundByClause i refuted let es2 = vs2 |> List.map (fun v -> @@ -907,25 +904,25 @@ let CompilePatternBasic | None -> rhs' - /// Select the set of discriminators which we can handle in one test, or as a series of - /// iterated tests, e.g. in the case of TPat_isinst. Ensure we only take at most one class of `TPat_query` at a time. + /// Select the set of discriminators which we can handle in one test, or as a series of iterated tests, + /// e.g. in the case of TPat_isinst. Ensure we only take at most one class of `TPat_query` at a time. /// Record the rule numbers so we know which rule the TPat_query cam from, so that when we project through /// the frontier we only project the right rule. and ChooseSimultaneousEdges frontiers path = frontiers |> chooseSimultaneousEdgeSet None (fun prevOpt (Frontier (i', active', _)) -> - if isMemOfActives path active' then - let p = lookupActive path active' |> snd - match getDiscrimOfPattern p with - | Some discrim -> - if (match prevOpt with None -> true | Some (EdgeDiscrim(_, discrimPrev, _)) -> discrimsHaveSameSimultaneousClass g discrim discrimPrev) then - Some (EdgeDiscrim(i', discrim, p.Range)), true - else - None, false - - | None -> - None, true - else - None, true) + if isMemOfActives path active' then + let _, p = lookupActive path active' + match getDiscrimOfPattern p with + | Some discrim -> + if (match prevOpt with None -> true | Some (EdgeDiscrim(_, discrimPrev, _)) -> discrimsHaveSameSimultaneousClass g discrim discrimPrev) then + Some (EdgeDiscrim(i', discrim, p.Range)), true + else + None, false + + | None -> + None, true + else + None, true) and IsCopyableInputExpr origInputExpr = match origInputExpr with @@ -1298,13 +1295,13 @@ let CompilePatternBasic mkFrontiers investigations i) |> List.concat) @ - mkFrontiers [([], ValMap<_>.Empty)] nclauses) + mkFrontiers [([], ValMap<_>.Empty)] nClauses) let dtree = InvestigateFrontiers [] frontiers - let targets = mbuilder.CloseTargets() + let targets = matchBuilder.CloseTargets() // Report unused targets @@ -1320,13 +1317,13 @@ let isPartialOrWhenClause (c: TypedMatchClause) = isPatternPartial c.Pattern || let rec CompilePattern g denv amap exprm matchm warnOnUnused actionOnFailure (origInputVal, origInputValTypars, origInputExprOpt) (clausesL: TypedMatchClause list) inputTy resultTy = - match clausesL with - | _ when List.exists isPartialOrWhenClause clausesL -> + match clausesL with + | _ when List.exists isPartialOrWhenClause clausesL -> // Partial clauses cause major code explosion if treated naively // Hence treat any pattern matches with any partial clauses clause-by-clause // First make sure we generate at least some of the obvious incomplete match warnings. - let warnOnUnused = false in (* we can't turn this on since we're pretending all partial's fail in order to control the complexity of this. *) + let warnOnUnused = false // we can't turn this on since we're pretending all partials fail in order to control the complexity of this. let warnOnIncomplete = true let clausesPretendAllPartialFail = List.collect (fun (TClause(p, whenOpt, tg, m)) -> [TClause(erasePartialPatterns p, whenOpt, tg, m)]) clausesL let _ = CompilePatternBasic g denv amap exprm matchm warnOnUnused warnOnIncomplete actionOnFailure (origInputVal, origInputValTypars, origInputExprOpt) clausesPretendAllPartialFail inputTy resultTy @@ -1334,19 +1331,18 @@ let rec CompilePattern g denv amap exprm matchm warnOnUnused actionOnFailure (o let rec atMostOnePartialAtATime clauses = match List.takeUntil isPartialOrWhenClause clauses with - | l, [] -> + | l, [] -> CompilePatternBasic g denv amap exprm matchm warnOnUnused warnOnIncomplete actionOnFailure (origInputVal, origInputValTypars, origInputExprOpt) l inputTy resultTy | l, (h :: t) -> - // Add the partial clause + // Add the partial clause. doGroupWithAtMostOnePartial (l @ [h]) t and doGroupWithAtMostOnePartial group rest = + // Compile the remaining clauses. + let decisionTree, targets = atMostOnePartialAtATime rest - // Compile the remaining clauses - let dtree, targets = atMostOnePartialAtATime rest - - // Make the expression that represents the remaining cases of the pattern match - let expr = mkAndSimplifyMatch NoSequencePointAtInvisibleBinding exprm matchm resultTy dtree targets + // Make the expression that represents the remaining cases of the pattern match. + let expr = mkAndSimplifyMatch NoSequencePointAtInvisibleBinding exprm matchm resultTy decisionTree targets // If the remainder of the match boiled away to nothing interesting. // We measure this simply by seeing if the range of the resulting expression is identical to matchm. @@ -1362,5 +1358,5 @@ let rec CompilePattern g denv amap exprm matchm warnOnUnused actionOnFailure (o atMostOnePartialAtATime clausesL - | _ -> - CompilePatternBasic g denv amap exprm matchm warnOnUnused true actionOnFailure (origInputVal, origInputValTypars, origInputExprOpt) clausesL inputTy resultTy + | _ -> + CompilePatternBasic g denv amap exprm matchm warnOnUnused true actionOnFailure (origInputVal, origInputValTypars, origInputExprOpt) clausesL inputTy resultTy diff --git a/src/fsharp/PatternMatchCompilation.fsi b/src/fsharp/PatternMatchCompilation.fsi index 8394611a16f..b35b5f42b27 100644 --- a/src/fsharp/PatternMatchCompilation.fsi +++ b/src/fsharp/PatternMatchCompilation.fsi @@ -9,7 +9,6 @@ open FSharp.Compiler.Tastops open FSharp.Compiler.TcGlobals open FSharp.Compiler.Range - /// What should the decision tree contain for any incomplete match? type ActionOnFailure = | ThrowIncompleteMatchException @@ -23,19 +22,20 @@ type ActionOnFailure = type Pattern = | TPat_const of Const * range | TPat_wild of range - | TPat_as of Pattern * PatternValBinding * range - | TPat_disjs of Pattern list * range - | TPat_conjs of Pattern list * range + | TPat_as of Pattern * PatternValBinding * range + | TPat_disjs of Pattern list * range + | TPat_conjs of Pattern list * range | TPat_query of (Expr * TType list * (ValRef * TypeInst) option * int * PrettyNaming.ActivePatternInfo) * Pattern * range | TPat_unioncase of UnionCaseRef * TypeInst * Pattern list * range | TPat_exnconstr of TyconRef * Pattern list * range - | TPat_tuple of TupInfo * Pattern list * TType list * range - | TPat_array of Pattern list * TType * range + | TPat_tuple of TupInfo * Pattern list * TType list * range + | TPat_array of Pattern list * TType * range | TPat_recd of TyconRef * TypeInst * Pattern list * range | TPat_range of char * char * range | TPat_null of range | TPat_isinst of TType * TType * PatternValBinding option * range - member Range : range + + member Range: range and PatternValBinding = | PBind of Val * TypeScheme @@ -43,10 +43,10 @@ and PatternValBinding = and TypedMatchClause = | TClause of Pattern * Expr option * DecisionTreeTarget * range -val ilFieldToTastConst : ILFieldInit -> Tast.Const +val ilFieldToTastConst: ILFieldInit -> Tast.Const /// Compile a pattern into a decision tree and a set of targets. -val internal CompilePattern : +val internal CompilePattern: TcGlobals -> DisplayEnv -> Import.ImportMap -> @@ -66,8 +66,8 @@ val internal CompilePattern : TType -> // result type TType -> - // produce TAST nodes - DecisionTree * DecisionTreeTarget list + // produce TAST nodes + DecisionTree * DecisionTreeTarget list exception internal MatchIncomplete of bool * (string * bool) option * range exception internal RuleNeverMatched of range From cec5b256f29f33e27d37fb3c27e4614d4cf8df61 Mon Sep 17 00:00:00 2001 From: Phillip Carter Date: Mon, 17 Jun 2019 17:48:19 -0700 Subject: [PATCH 05/11] Remove IVT to fsi interactive settings (#6998) --- .../FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj index 9b24fac60de..443f6c6efd2 100644 --- a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj +++ b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj @@ -42,7 +42,6 @@ - From ce98492ff47d27ff2134d4da30597786c2582788 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 18 Jun 2019 12:34:19 -0700 Subject: [PATCH 06/11] properly fail the build on unix failures (#7015) I discovered that unix (linux/macos) failures would report the failing tests via the `Tests` tab in ADO, but the specific build step would appear green which made diagnosing failures difficult. --- eng/build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/build.sh b/eng/build.sh index c467296d3d3..fceb485349a 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -176,7 +176,11 @@ function TestUsingNUnit() { projectname="${projectname%.*}" testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml" args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"nunit;LogFilePath=$testlogpath\"" - "$DOTNET_INSTALL_DIR/dotnet" $args + "$DOTNET_INSTALL_DIR/dotnet" $args || { + local exit_code=$? + echo "dotnet test failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code + } } function BuildSolution { From cb880d582d76c72087722a4b3e0cff17e7ac0f91 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 18 Jun 2019 16:32:29 -0700 Subject: [PATCH 07/11] Fixed line directive ranges when not applying a line directive (#7011) * Fixed line directive ranges when not applying a line directive * Update ProjectAnalysisTests.fs --- src/fsharp/lex.fsl | 10 +++++++--- tests/service/ProjectAnalysisTests.fs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index 0af96d71c9a..c32c9a641b6 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -524,10 +524,14 @@ rule token args skip = parse // Construct the new position if args.applyLineDirectives then lexbuf.EndPos <- pos.ApplyLineDirective((match file with Some f -> fileIndexOfFile f | None -> pos.FileIndex), line) - + else + // add a newline when we don't apply a directive since we consumed a newline getting here + newline lexbuf token args skip lexbuf - else - if not skip then (HASH_LINE (LexCont.Token !args.ifdefStack)) else token args skip lexbuf } + else + // add a newline when we don't apply a directive since we consumed a newline getting here + newline lexbuf + (HASH_LINE (LexCont.Token !args.ifdefStack)) } | "<@" { checkExprOp lexbuf; LQUOTE ("<@ @>", false) } | "<@@" { checkExprOp lexbuf; LQUOTE ("<@@ @@>", true) } diff --git a/tests/service/ProjectAnalysisTests.fs b/tests/service/ProjectAnalysisTests.fs index 16ef433e691..c9635c70830 100644 --- a/tests/service/ProjectAnalysisTests.fs +++ b/tests/service/ProjectAnalysisTests.fs @@ -5291,7 +5291,7 @@ let ``Test line directives in foreground analysis`` () = // see https://github.c for e in checkResults1.Errors do printfn "ProjectLineDirectives checkResults1 error file: <<<%s>>>" e.FileName - [ for e in checkResults1.Errors -> e.StartLineAlternate, e.EndLineAlternate, e.FileName ] |> shouldEqual [(4, 4, ProjectLineDirectives.fileName1)] + [ for e in checkResults1.Errors -> e.StartLineAlternate, e.EndLineAlternate, e.FileName ] |> shouldEqual [(5, 5, ProjectLineDirectives.fileName1)] //------------------------------------------------------ From b90096809218283f325143708914de2658cd2dc1 Mon Sep 17 00:00:00 2001 From: Phillip Carter Date: Wed, 19 Jun 2019 02:22:53 -0700 Subject: [PATCH 08/11] Remove IVTs to legacy language service (#7001) * Remove IVTs to legacy language service * Use public API and remove seemingly useless test --- .../FSharp.Compiler.Private.fsproj | 2 -- .../FSharp.LanguageService/GotoDefinition.fs | 13 ++++++------- .../FSharp.LanguageService/Intellisense.fs | 2 +- .../Tests.LanguageService.General.fs | 19 ------------------- 4 files changed, 7 insertions(+), 29 deletions(-) diff --git a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj index 443f6c6efd2..d468a8a9968 100644 --- a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj +++ b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj @@ -40,8 +40,6 @@ - - diff --git a/vsintegration/src/FSharp.LanguageService/GotoDefinition.fs b/vsintegration/src/FSharp.LanguageService/GotoDefinition.fs index ca720f0927f..aa4e9441f8a 100644 --- a/vsintegration/src/FSharp.LanguageService/GotoDefinition.fs +++ b/vsintegration/src/FSharp.LanguageService/GotoDefinition.fs @@ -5,15 +5,10 @@ namespace Microsoft.VisualStudio.FSharp.LanguageService open System -open System.IO -open System.Collections.Generic open System.Diagnostics open Microsoft.VisualStudio -open Microsoft.VisualStudio.Shell -open Microsoft.VisualStudio.Shell.Interop open Microsoft.VisualStudio.TextManager.Interop open FSharp.Compiler -open FSharp.Compiler.Lib open FSharp.Compiler.SourceCodeServices module internal OperatorToken = @@ -21,7 +16,9 @@ module internal OperatorToken = let asIdentifier_DEPRECATED (token : TokenInfo) = // Typechecker reports information about all values in the same fashion no matter whether it is named value (let binding) or operator // here we piggyback on this fact and just pretend that we need data time for identifier - let tagOfIdentToken = FSharp.Compiler.Parser.tagOfToken(FSharp.Compiler.Parser.IDENT "") + + let tagOfIdentToken = FSharpTokenTag.IDENT + let endCol = token.EndIndex + 1 // EndIndex from GetTokenInfoAt points to the last operator char, but here it should point to column 'after' the last char tagOfIdentToken, token.StartIndex, endCol @@ -69,7 +66,9 @@ module internal GotoDefinition = |> GotoDefinitionResult_DEPRECATED.MakeError | Some(colIdent, tag, qualId) -> if typedResults.HasFullTypeCheckInfo then - if Parser.tokenTagToTokenId tag <> Parser.TOKEN_IDENT then + // Used to be the Parser's internal definition, now hard-coded to avoid an IVT into the parser itsef. + // Dead code (aside from legacy tests), ignore + if tag <> FSharpTokenTag.IDENT then Strings.GotoDefinitionFailed_NotIdentifier() |> GotoDefinitionResult_DEPRECATED.MakeError else diff --git a/vsintegration/src/FSharp.LanguageService/Intellisense.fs b/vsintegration/src/FSharp.LanguageService/Intellisense.fs index b380bf13e20..dbf9d7fa5ab 100644 --- a/vsintegration/src/FSharp.LanguageService/Intellisense.fs +++ b/vsintegration/src/FSharp.LanguageService/Intellisense.fs @@ -224,7 +224,7 @@ type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: // We intercept this call only to get the initial extent // of what was committed to the source buffer. let result = decl.GetName(filterText, index) - FSharp.Compiler.Lexhelp.Keywords.QuoteIdentifierIfNeeded result + Keywords.QuoteIdentifierIfNeeded result override decl.IsCommitChar(commitCharacter) = // Usual language identifier rules... diff --git a/vsintegration/tests/UnitTests/LegacyLanguageService/Tests.LanguageService.General.fs b/vsintegration/tests/UnitTests/LegacyLanguageService/Tests.LanguageService.General.fs index 47e8e3b1134..6bc62eda151 100644 --- a/vsintegration/tests/UnitTests/LegacyLanguageService/Tests.LanguageService.General.fs +++ b/vsintegration/tests/UnitTests/LegacyLanguageService/Tests.LanguageService.General.fs @@ -101,25 +101,6 @@ type UsingMSBuild() = n ) 0 - - [] - member public this.``PublicSurfaceArea.DotNetReflection``() = - let ps = publicTypesInAsm @"FSharp.ProjectSystem.FSharp.dll" - Assert.AreEqual(1, ps) // BuildPropertyDescriptor - let ls = publicTypesInAsm @"FSharp.LanguageService.dll" - Assert.AreEqual(0, ls) - let compis = publicTypesInAsm @"FSharp.Compiler.Interactive.Settings.dll" - Assert.AreEqual(5, compis) - let compserver = publicTypesInAsm @"FSharp.Compiler.Server.Shared.dll" - Assert.AreEqual(0, compserver) - let lsbase = publicTypesInAsm @"FSharp.LanguageService.Base.dll" - Assert.AreEqual(0, lsbase) - let psbase = publicTypesInAsm @"FSharp.ProjectSystem.Base.dll" - Assert.AreEqual(17, psbase) - let fsi = publicTypesInAsm @"FSharp.VS.FSI.dll" - Assert.AreEqual(1, fsi) - - [] member public this.``ReconcileErrors.Test1``() = let (_solution, project, file) = this.CreateSingleFileProject(["erroneous"]) From 66a209afac4f9b0dd041bc99aed49d68a78b1990 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 19 Jun 2019 15:02:44 -0700 Subject: [PATCH 09/11] [master] Update dependencies from dotnet/arcade (#7013) * Update dependencies from https://github.com/dotnet/arcade build 20190618.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19318.2 * Update dependencies from https://github.com/dotnet/arcade build 20190619.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19319.1 --- eng/Version.Details.xml | 4 +- eng/common/PSScriptAnalyzerSettings.psd1 | 11 ++ eng/common/internal-feed-operations.ps1 | 135 +++++++++++++++++ eng/common/internal-feed-operations.sh | 142 ++++++++++++++++++ .../templates/job/publish-build-assets.yml | 1 - .../channels/public-dev-release.yml | 3 +- .../channels/public-validation-release.yml | 3 +- .../post-build/setup-maestro-vars.yml | 28 +++- global.json | 2 +- 9 files changed, 313 insertions(+), 16 deletions(-) create mode 100644 eng/common/PSScriptAnalyzerSettings.psd1 create mode 100644 eng/common/internal-feed-operations.ps1 create mode 100644 eng/common/internal-feed-operations.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0091420aa13..d4aef6fd563 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - aa4285be7fab64e2b6e62e4d5688ea50931c407c + 209388b8a700bb99690ae7fbbabe7e55f1999a8f diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 00000000000..4c1ea7c98ea --- /dev/null +++ b/eng/common/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,11 @@ +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') +} \ No newline at end of file diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 new file mode 100644 index 00000000000..8b8bafd6a89 --- /dev/null +++ b/eng/common/internal-feed-operations.ps1 @@ -0,0 +1,135 @@ +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host "Setting up Cred Provider NuGet plugin in the agent..." + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host "Installing plugin..." + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-Host "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + } + else { + Write-Host "CredProvider plugin was installed correctly!" + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = "$RepoRoot\NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-Host "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host "No internal endpoints found in NuGet.config" + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host "Arcade SDK restored!" + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like "setup") { + SetupCredProvider $AuthToken + } + elseif ($Operation -like "install-restore") { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-Host "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh new file mode 100644 index 00000000000..1ff654d2ffc --- /dev/null +++ b/eng/common/internal-feed-operations.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + echo "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="$repo_root/NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + echo "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 619ec68aa43..01442216816 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -57,7 +57,6 @@ jobs: /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:Configuration=$(_BuildConfig) - /v:detailed condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - task: powershell@2 diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index b332cb51732..c61eaa927d9 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -138,8 +138,7 @@ stages: inputs: targetType: inline script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) - continueOnError: true + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location - template: ../promote-build.yml parameters: diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 0b9719da82b..23725c6d620 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -84,8 +84,7 @@ stages: inputs: targetType: inline script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) - continueOnError: true + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location - template: ../promote-build.yml parameters: diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index b40e0260a3a..9de585a94ac 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -16,11 +16,23 @@ jobs: inputs: targetType: inline script: | - . "$(Build.SourcesDirectory)/eng/common/tools.ps1" - - $BarId = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/BARBuildId.txt" - Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId - - $Channels = "" - Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/Channels.txt" | ForEach-Object { $Channels += "$_ ," } - Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + # This is needed to make Write-PipelineSetVariable works in this context + if ($env:BUILD_BUILDNUMBER -ne "" -and $env:BUILD_BUILDNUMBER -ne $null) { + $ci = $true + + . "$(Build.SourcesDirectory)/eng/common/tools.ps1" + + $BarId = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/BARBuildId.txt" + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + + Write-Host "Asked Write-PipelineSetVariable to create BARBuildId with value '$BarId'" + + $Channels = "" + Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/Channels.txt" | ForEach-Object { $Channels += "$_ ," } + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + + Write-Host "Asked Write-PipelineSetVariable to create InitialChannels with value '$Channels'" + } + else { + Write-Host "This step can only be run in an Azure DevOps CI environment." + } diff --git a/global.json b/global.json index ace8d4fc826..586531e4861 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19315.2", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19319.1", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From 51684d453cc08f63f93a5646604e22737e553e41 Mon Sep 17 00:00:00 2001 From: Phillip Carter Date: Wed, 19 Jun 2019 22:44:35 -0700 Subject: [PATCH 10/11] Add get/set to item description tooltip (#7016) * Add get/set to item description tooltip * Fix QuickInfo test * Adjust gettersetter layout * Tag the keyword again --- src/fsharp/NicePrint.fs | 14 +++++++++++++- vsintegration/tests/UnitTests/QuickInfoTests.fs | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/fsharp/NicePrint.fs b/src/fsharp/NicePrint.fs index ab45f12c137..5d56543e992 100644 --- a/src/fsharp/NicePrint.fs +++ b/src/fsharp/NicePrint.fs @@ -1405,12 +1405,24 @@ module InfoMemberPrinting = | None -> tagProperty | Some vref -> tagProperty >> mkNav vref.DefinitionRange let nameL = DemangleOperatorNameAsLayout tagProp pinfo.PropertyName + let getterSetter = + match pinfo.HasGetter, pinfo.HasSetter with + | (true, false) -> + wordL (tagKeyword "with") ^^ wordL (tagText "get") + | (false, true) -> + wordL (tagKeyword "with") ^^ wordL (tagText "set") + | (true, true) -> + wordL (tagKeyword "with") ^^ wordL (tagText "get, set") + | (false, false) -> + emptyL + wordL (tagText (FSComp.SR.typeInfoProperty())) ^^ layoutTyconRef denv pinfo.ApparentEnclosingTyconRef ^^ SepL.dot ^^ nameL ^^ RightL.colon ^^ - layoutType denv rty + layoutType denv rty ^^ + getterSetter let formatMethInfoToBufferFreeStyle amap m denv os (minfo: MethInfo) = let _, resL = prettyLayoutOfMethInfoFreeStyle amap m denv emptyTyparInst minfo diff --git a/vsintegration/tests/UnitTests/QuickInfoTests.fs b/vsintegration/tests/UnitTests/QuickInfoTests.fs index 19d140de88b..bfa6c7df729 100644 --- a/vsintegration/tests/UnitTests/QuickInfoTests.fs +++ b/vsintegration/tests/UnitTests/QuickInfoTests.fs @@ -440,6 +440,6 @@ module Test = } """ let quickInfo = GetQuickInfoTextFromCode code - let expected = "property MyDistance.asNautical: MyDistance" + let expected = "property MyDistance.asNautical: MyDistance with get" Assert.AreEqual(expected, quickInfo) () From fa5ef919b8ee2a1604538ade5dce8e1a0e977ae8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2019 10:01:41 -0700 Subject: [PATCH 11/11] [master] Update dependencies from dotnet/arcade (#7023) * Update dependencies from https://github.com/dotnet/arcade build 20190619.25 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19319.25 * Update dependencies from https://github.com/dotnet/arcade build 20190620.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19320.1 --- eng/Version.Details.xml | 4 +- eng/common/sdl/execute-all-sdl-tools.ps1 | 56 ++++++++++++------------ eng/common/sdl/init-sdl.ps1 | 6 +-- eng/common/sdl/push-gdn.ps1 | 6 +-- eng/common/templates/job/execute-sdl.yml | 37 ++++++++++++++++ global.json | 2 +- 6 files changed, 74 insertions(+), 37 deletions(-) create mode 100644 eng/common/templates/job/execute-sdl.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d4aef6fd563..3f53c799b29 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 209388b8a700bb99690ae7fbbabe7e55f1999a8f + b21c24996a73aa62b7a1ee69f546b9d2eb084f29 diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 74080f22d1e..0635f26fb63 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -1,28 +1,28 @@ Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI pacakge (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName="master", # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory, # Required: the directory where source files are located - [string] $ArtifactsDirectory, # Required: the directory where build artifacts are located - [string] $DncEngAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCHNAME, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error ) $ErrorActionPreference = "Stop" @@ -51,7 +51,7 @@ if ($ValidPath -eq $False) exit 1 } -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -DncEngAccessToken $DncEngAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel $gdnFolder = Join-Path $ArtifactsDirectory ".gdn" if ($TsaOnboard) { @@ -69,14 +69,14 @@ if ($TsaOnboard) { } if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel } if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel } if ($UpdateBaseline) { - & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Update baseline" + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" } if ($TsaPublish) { diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index cbf5c36a8f7..26e01c0673c 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -3,7 +3,7 @@ Param( [string] $Repository, [string] $BranchName="master", [string] $WorkingDirectory, - [string] $DncEngAccessToken, + [string] $AzureDevOpsAccessToken, [string] $GuardianLoggerLevel="Standard" ) @@ -12,7 +12,7 @@ Set-StrictMode -Version 2.0 $LASTEXITCODE = 0 # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$DncEngAccessToken")) +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") $uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" $zipFile = "$WorkingDirectory/gdn.zip" @@ -44,5 +44,5 @@ Try if ($LASTEXITCODE -ne 0) { Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." } - & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Initialize gdn folder" + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" } \ No newline at end of file diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index cacaf8e9127..79c707d6d8a 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -2,7 +2,7 @@ Param( [string] $Repository, [string] $BranchName="master", [string] $GdnFolder, - [string] $DncEngAccessToken, + [string] $AzureDevOpsAccessToken, [string] $PushReason ) @@ -16,8 +16,8 @@ if (Test-Path $sdlDir) { Remove-Item -Force -Recurse $sdlDir } -Write-Host "git clone https://dnceng:`$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" -git clone https://dnceng:$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir if ($LASTEXITCODE -ne 0) { Write-Error "Git clone failed with exit code $LASTEXITCODE." } diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml new file mode 100644 index 00000000000..2f669fd95ba --- /dev/null +++ b/eng/common/templates/job/execute-sdl.yml @@ -0,0 +1,37 @@ +parameters: + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + continueOnError: false # optional: determines whether to continue the build if the step errors; + dependsOn: '' # Optional: dependencies of the job + +jobs: +- job: Run_SDL + dependsOn: ${{ parameters.dependsOn }} + displayName: Run SDL tool + variables: + - group: DotNet-VSTS-Bot + steps: + - checkout: self + clean: true + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + - task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + - ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 + -GuardianPackageName Microsoft.Guardian.Cli.0.3.2 + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} diff --git a/global.json b/global.json index 586531e4861..899ac158788 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19319.1", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19320.1", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } }