From 002dd41474b7fbc0061e72079175e03a87ac4aba Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 2 May 2019 12:45:00 +0000 Subject: [PATCH 01/37] Update dependencies from https://github.com/dotnet/arcade build 20190501.6 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19251.6 --- eng/Version.Details.xml | 4 ++-- eng/common/build.ps1 | 4 ++++ eng/common/build.sh | 4 ++++ eng/common/init-tools-native.ps1 | 6 +++++- eng/common/init-tools-native.sh | 12 ++++-------- eng/common/tools.ps1 | 10 ++++++++++ eng/common/tools.sh | 12 ++++++++++++ global.json | 4 ++-- 8 files changed, 43 insertions(+), 13 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8d489b2804..3eeff155fd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ac8d88df02d246d3147338fcfb03b1b93dc84b53 + ea9b2530a1559dd6426730aba52fca1924db2b65 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index d0147db4bd..ed9f754ea8 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -122,6 +122,10 @@ try { . $configureToolsetScript } + if ($restore) { + InitializeNativeTools + } + Build } catch { diff --git a/eng/common/build.sh b/eng/common/build.sh index 40b1e8ec73..e14210632a 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -218,4 +218,8 @@ if [[ -n "${useInstalledDotNetCli:-}" ]]; then use_installed_dotnet_cli="$useInstalledDotNetCli" fi +if [[ "$restore" == true ]]; then + InitializeNativeTools +fi + Build diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index 495a563a75..a4306bd37e 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -41,9 +41,13 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile = "$PSScriptRoot\..\..\global.json" + [string] $GlobalJsonFile ) +if (!$GlobalJsonFile) { + $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json" +} + Set-StrictMode -version 2.0 $ErrorActionPreference="Stop" diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 54b70f678b..fc72d13948 100644 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -9,7 +9,7 @@ clean=false force=false download_retries=5 retry_wait_time_seconds=30 -global_json_file="${scriptroot}/../../global.json" +global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" declare -A native_assets . $scriptroot/native/common-library.sh @@ -71,6 +71,7 @@ function ReadGlobalJsonNativeTools { local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') native_tools_list=${native_tools_list//[\" ]/} native_tools_list=${native_tools_list//,/$'\n'} + native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')" local old_IFS=$IFS while read -r line; do @@ -116,8 +117,6 @@ else installer_command+=" --clean" fi - echo "Installing $tool version $tool_version" - echo "Executing '$installer_command'" $installer_command if [[ $? != 0 ]]; then @@ -127,19 +126,16 @@ else done fi -if [[ ! -z $clean ]]; then +if [[ $clean = true ]]; then exit 0 fi if [[ -d $install_bin ]]; then echo "Native tools are available from $install_bin" - if [[ !-z BUILD_BUILDNUMBER ]]; then - echo "##vso[task.prependpath]$install_bin" - fi + echo "##vso[task.prependpath]$install_bin" else echo "Native tools install directory does not exist, installation failed" >&2 exit 1 fi exit 0 - diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5c4a129c86..09794dff83 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -391,6 +391,16 @@ function GetSdkTaskProject([string]$taskName) { return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" } +function InitializeNativeTools() { + if (Get-Member -InputObject $GlobalJson -Name "native-tools") { + $nativeArgs="" + if ($ci) { + $nativeArgs = "-InstallDirectory $ToolsDir" + } + Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" + } +} + function InitializeToolset() { if (Test-Path variable:global:_ToolsetBuildProj) { return $global:_ToolsetBuildProj diff --git a/eng/common/tools.sh b/eng/common/tools.sh index ecdece1f85..59f47c5fa9 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -212,6 +212,17 @@ function GetNuGetPackageCachePath { _GetNuGetPackageCachePath=$NUGET_PACKAGES } +function InitializeNativeTools() { + if grep -Fq "native-tools" $global_json_file + then + local nativeArgs="" + if [[ "$ci" == true ]]; then + nativeArgs="-InstallDirectory $tools_dir" + fi + "$_script_dir/init-tools-native.sh" $nativeArgs + fi +} + function InitializeToolset { if [[ -n "${_InitializeToolset:-}" ]]; then return @@ -307,6 +318,7 @@ eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` artifacts_dir="$repo_root/artifacts" toolset_dir="$artifacts_dir/toolset" +tools_dir="$repo_root/.tools" log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" diff --git a/global.json b/global.json index cd90071adc..e497317ff8 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "tools": { - "dotnet": "2.2.103", + "dotnet": "2.2.203", "vs": { "version": "15.9" }, "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19230.6" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19251.6" } } From 2ebe38ddf0a3c8af55e311e36a3f381278156ae1 Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Thu, 2 May 2019 07:16:02 -0700 Subject: [PATCH 02/37] update core 1.1 version --- eng/restore-toolset.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/restore-toolset.ps1 b/eng/restore-toolset.ps1 index a1c99361b3..21b799ae50 100644 --- a/eng/restore-toolset.ps1 +++ b/eng/restore-toolset.ps1 @@ -16,7 +16,7 @@ function InitializeCustomSDKToolset { $env:DOTNET_CLI_TELEMETRY_PROFILE='$env:DOTNET_CLI_TELEMETRY_PROFILE;https://github.com/dotnet/roslyn-sdk' $cli = InitializeDotnetCli -install:$true - InstallDotNetSharedFramework "1.1.10" + InstallDotNetSharedFramework "1.1.12" } function InstallDotNetSharedFramework([string]$version) { From afdb812d6d286463e0862de3d2d63dcc67b28c82 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 3 May 2019 12:45:33 +0000 Subject: [PATCH 03/37] Update dependencies from https://github.com/dotnet/arcade build 20190502.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19252.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3eeff155fd..6833196a77 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ea9b2530a1559dd6426730aba52fca1924db2b65 + ed63a7faf2d119829dd971be91bb46b6ebaa1dd2 diff --git a/global.json b/global.json index e497317ff8..befbb5982c 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19251.6" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19252.2" } } From 78778e553b049bb2c3a8a586c84dba238ca22f22 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 4 May 2019 12:41:56 +0000 Subject: [PATCH 04/37] Update dependencies from https://github.com/dotnet/arcade build 20190503.8 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19253.8 --- eng/Version.Details.xml | 4 ++-- eng/common/build.ps1 | 2 +- eng/common/build.sh | 2 +- global.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6833196a77..eb457847e3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ed63a7faf2d119829dd971be91bb46b6ebaa1dd2 + 4cacf7aa7b2cd8bdd042e721e6ad73d809dd5678 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index ed9f754ea8..d7e3799ebd 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -122,7 +122,7 @@ try { . $configureToolsetScript } - if ($restore) { + if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) { InitializeNativeTools } diff --git a/eng/common/build.sh b/eng/common/build.sh index e14210632a..d038959ab4 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -218,7 +218,7 @@ if [[ -n "${useInstalledDotNetCli:-}" ]]; then use_installed_dotnet_cli="$useInstalledDotNetCli" fi -if [[ "$restore" == true ]]; then +if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then InitializeNativeTools fi diff --git a/global.json b/global.json index befbb5982c..d913d2cdce 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19252.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19253.8" } } From 5e452a216acdf3701c7eebe190b210108c21006d Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sun, 5 May 2019 12:41:40 +0000 Subject: [PATCH 05/37] Update dependencies from https://github.com/dotnet/arcade build 20190504.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19254.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index eb457847e3..b45807d105 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 4cacf7aa7b2cd8bdd042e721e6ad73d809dd5678 + 1b8589bbf53b9a5e819460798eff59830f39a3be diff --git a/global.json b/global.json index d913d2cdce..3f6b62885e 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19253.8" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19254.1" } } From 0eb8bfb1d3d693bd9421e76dae32a98908baaf8a Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Mon, 6 May 2019 12:47:53 +0000 Subject: [PATCH 06/37] Update dependencies from https://github.com/dotnet/arcade build 20190505.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19255.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b45807d105..f9036d33d6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 1b8589bbf53b9a5e819460798eff59830f39a3be + 6a34948f7bdbc3ceb2fb16441b49f7748a462646 diff --git a/global.json b/global.json index 3f6b62885e..8540277b63 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19254.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19255.2" } } From 54f666a38716e0df1e047809d8ed153cb63a0e52 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Tue, 7 May 2019 12:53:47 +0000 Subject: [PATCH 07/37] Update dependencies from https://github.com/dotnet/arcade build 20190506.12 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19256.12 --- eng/Version.Details.xml | 4 +-- eng/common/build.sh | 18 ++---------- eng/common/dotnet-install.cmd | 2 ++ eng/common/dotnet-install.ps1 | 22 ++++++++++++++ eng/common/dotnet-install.sh | 49 ++++++++++++++++++++++++++++++++ eng/common/templates/job/job.yml | 3 ++ eng/common/tools.ps1 | 27 ++++++++++++++---- eng/common/tools.sh | 39 ++++++++++++++++++++----- global.json | 2 +- 9 files changed, 134 insertions(+), 32 deletions(-) create mode 100644 eng/common/dotnet-install.cmd create mode 100644 eng/common/dotnet-install.ps1 create mode 100644 eng/common/dotnet-install.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f9036d33d6..ae3a177455 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 6a34948f7bdbc3ceb2fb16441b49f7748a462646 + c31fac9f6899094226cb5cd77c85b8f60ecafa3d diff --git a/eng/common/build.sh b/eng/common/build.sh index d038959ab4..ce846d888d 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -35,7 +35,7 @@ usage() echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" echo "" - echo "Command line arguments starting with '/p:' are passed through to MSBuild." + echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." } @@ -137,22 +137,8 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; - -p:*|/p:*) - properties="$properties $1" - ;; - -m:*|/m:*) - properties="$properties $1" - ;; - -bl:*|/bl:*) - properties="$properties $1" - ;; - -dl:*|/dl:*) - properties="$properties $1" - ;; *) - echo "Invalid argument: $1" - usage - exit 1 + properties="$properties $1" ;; esac diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd new file mode 100644 index 0000000000..b1c2642e76 --- /dev/null +++ b/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 new file mode 100644 index 0000000000..5987943fd6 --- /dev/null +++ b/eng/common/dotnet-install.ps1 @@ -0,0 +1,22 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = "minimal", + [string] $architecture = "", + [string] $version = "Latest", + [string] $runtime = "dotnet" +) + +. $PSScriptRoot\tools.ps1 + +try { + $dotnetRoot = Join-Path $RepoRoot ".dotnet" + InstallDotNet $dotnetRoot $version $architecture $runtime $true +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh new file mode 100644 index 0000000000..c3072c958a --- /dev/null +++ b/eng/common/dotnet-install.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +version='Latest' +architecture='' +runtime='dotnet' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + shift +done + +. "$scriptroot/tools.sh" +dotnetRoot="$repo_root/.dotnet" +InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { + local exit_code=$? + echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 7839b70bb7..1814e0ab61 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -90,6 +90,9 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} variables: + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 09794dff83..d86eef1e3a 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -108,7 +108,7 @@ function InitializeDotNetCli([bool]$install) { } # Find the first path on %PATH% that contains the dotnet.exe - if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) { + if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent @@ -119,7 +119,7 @@ function InitializeDotNetCli([bool]$install) { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" @@ -152,7 +152,7 @@ function InitializeDotNetCli([bool]$install) { } function GetDotNetInstallScript([string] $dotnetRoot) { - $installScript = "$dotnetRoot\dotnet-install.ps1" + $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript @@ -162,9 +162,21 @@ function GetDotNetInstallScript([string] $dotnetRoot) { } function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { + InstallDotNet $dotnetRoot $version $architecture +} + +function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot $installScript = GetDotNetInstallScript $dotnetRoot - $archArg = if ($architecture) { $architecture } else { "" } - & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg + $installParameters = @{ + Version = $version + InstallDir = $dotnetRoot + } + + if ($architecture) { $installParameters.Architecture = $architecture } + if ($runtime) { $installParameters.Runtime = $runtime } + if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + + & $installScript @installParameters if ($lastExitCode -ne 0) { Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red ExitWithExitCode $lastExitCode @@ -429,6 +441,7 @@ function InitializeToolset() { $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } '' | Set-Content $proj + MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile $path = Get-Content $toolsetLocationFile -TotalCount 1 @@ -522,6 +535,8 @@ $ToolsDir = Join-Path $RepoRoot ".tools" $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json +# true if global.json contains a "runtimes" section +$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } Create-Directory $ToolsetDir Create-Directory $TempDir @@ -534,4 +549,4 @@ if ($ci) { $env:TEMP = $TempDir $env:TMP = $TempDir -} +} \ No newline at end of file diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 59f47c5fa9..9dc565e296 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -101,7 +101,7 @@ function InitializeDotNetCli { fi # Find the first path on $PATH that contains the dotnet.exe - if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` if [[ -n "$dotnet_path" ]]; then ResolvePath "$dotnet_path" @@ -115,10 +115,11 @@ function InitializeDotNetCli { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then + if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else dotnet_root="$repo_root/.dotnet" + export DOTNET_INSTALL_DIR="$dotnet_root" if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then @@ -149,16 +150,34 @@ function InitializeDotNetCli { function InstallDotNetSdk { local root=$1 local version=$2 + local architecture="" + if [[ $# == 3 ]]; then + architecture=$3 + fi + InstallDotNet "$root" "$version" $architecture +} +function InstallDotNet { + local root=$1 + local version=$2 + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local arch_arg="" - if [[ $# == 3 ]]; then - arch_arg="--architecture $3" + local archArg='' + if [[ "$#" -ge "3" ]]; then + archArg="--architecture $3" + fi + local runtimeArg='' + if [[ "$#" -ge "4" ]]; then + runtimeArg="--runtime $4" fi - bash "$install_script" --version $version --install-dir "$root" $arch_arg || { + local skipNonVersionedFilesArg="" + if [[ "$#" -ge "5" ]]; then + skipNonVersionedFilesArg="--skip-non-versioned-files" + fi + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code @@ -323,6 +342,12 @@ log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" +# determine if global.json contains a "runtimes" entry +global_json_has_runtimes=false +dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +if [[ -n "$dotnetlocal_key" ]]; then + global_json_has_runtimes=true +fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then @@ -337,4 +362,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi +fi \ No newline at end of file diff --git a/global.json b/global.json index 8540277b63..5e9039141e 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19255.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19256.12" } } From 7b0b7509349c8ffb74f0d08b26a7f521a52d4a77 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Wed, 8 May 2019 12:43:47 +0000 Subject: [PATCH 08/37] Update dependencies from https://github.com/dotnet/arcade build 20190507.7 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19257.7 --- eng/Version.Details.xml | 4 +- eng/common/cross/arm/sources.list.vivid | 11 ----- eng/common/cross/arm/sources.list.wily | 11 ----- eng/common/cross/arm64/sources.list.vivid | 11 ----- eng/common/cross/arm64/sources.list.wily | 11 ----- eng/common/cross/build-rootfs.sh | 58 ++++++++++++++--------- eng/common/cross/x86/sources.list.vivid | 11 ----- eng/common/cross/x86/sources.list.wily | 11 ----- global.json | 2 +- 9 files changed, 39 insertions(+), 91 deletions(-) delete mode 100644 eng/common/cross/arm/sources.list.vivid delete mode 100644 eng/common/cross/arm/sources.list.wily delete mode 100644 eng/common/cross/arm64/sources.list.vivid delete mode 100644 eng/common/cross/arm64/sources.list.wily delete mode 100644 eng/common/cross/x86/sources.list.vivid delete mode 100644 eng/common/cross/x86/sources.list.wily diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ae3a177455..bf0138e992 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - c31fac9f6899094226cb5cd77c85b8f60ecafa3d + bda52d7619f9420de46f2c39ffc972864bbcab63 diff --git a/eng/common/cross/arm/sources.list.vivid b/eng/common/cross/arm/sources.list.vivid deleted file mode 100644 index 0b1215e475..0000000000 --- a/eng/common/cross/arm/sources.list.vivid +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.wily b/eng/common/cross/arm/sources.list.wily deleted file mode 100644 index e23d1e02a0..0000000000 --- a/eng/common/cross/arm/sources.list.wily +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.vivid b/eng/common/cross/arm64/sources.list.vivid deleted file mode 100644 index 0b1215e475..0000000000 --- a/eng/common/cross/arm64/sources.list.vivid +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.wily b/eng/common/cross/arm64/sources.list.wily deleted file mode 100644 index e23d1e02a0..0000000000 --- a/eng/common/cross/arm64/sources.list.wily +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 805948ca83..83ec39195c 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -2,21 +2,21 @@ usage() { - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount]" + echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfs ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "LinuxCodeName - optional, Code name for Linux, can be: trusty(default), vivid, wily, xenial, zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo "lldbx.y - optional, LLDB version, can be: lldb3.6(default), lldb3.8, lldb3.9, lldb4.0, no-lldb. Ignored for alpine" + echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" echo "--skipunmount - optional, will skip the unmount of rootfs folder." exit 1 } -__LinuxCodeName=trusty +__LinuxCodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" -__LLDB_Package="lldb-3.6-dev" +__LLDB_Package="liblldb-3.9-dev" __SkipUnmount=0 # base development support @@ -53,8 +53,12 @@ __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" __UnprocessedBuildArgs= -for i in "$@" ; do - lowerI="$(echo $i | awk '{print tolower($0)}')" +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | awk '{print tolower($0)}')" case $lowerI in -?|-h|--help) usage @@ -95,38 +99,40 @@ for i in "$@" ; do lldb4.0) __LLDB_Package="liblldb-4.0-dev" ;; + lldb5.0) + __LLDB_Package="liblldb-5.0-dev" + ;; + lldb6.0) + __LLDB_Package="liblldb-6.0-dev" + ;; no-lldb) unset __LLDB_Package ;; - vivid) - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=vivid - fi - ;; - wily) + trusty) # Ubuntu 14.04 if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=wily + __LinuxCodeName=trusty fi ;; - xenial) + xenial) # Ubunry 16.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=xenial fi ;; - zesty) + zesty) # Ununtu 17.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=zesty fi ;; - bionic) + bionic) # Ubuntu 18.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=bionic fi ;; - jessie) + jessie) # Debian 8 __LinuxCodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; + # TBD Stretch -> Debian 9, Buster -> Debian 10 tizen) if [ "$__BuildArch" != "armel" ]; then echo "Tizen is available only for armel." @@ -144,10 +150,16 @@ for i in "$@" ; do --skipunmount) __SkipUnmount=1 ;; + --rootfsdir|-rootfsdir) + shift + __RootfsDir=$1 + ;; *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; esac + + shift done if [ "$__BuildArch" == "armel" ]; then @@ -155,12 +167,14 @@ if [ "$__BuildArch" == "armel" ]; then fi __UbuntuPackages+=" ${__LLDB_Package:-}" -__RootfsDir="$__CrossDir/rootfs/$__BuildArch" - -if [[ -n "$ROOTFS_DIR" ]]; then +if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then __RootfsDir=$ROOTFS_DIR fi +if [ -z "$__RootfsDir" ]; then + __RootfsDir="$__CrossDir/rootfs/$__BuildArch" +fi + if [ -d "$__RootfsDir" ]; then if [ $__SkipUnmount == 0 ]; then umount $__RootfsDir/* diff --git a/eng/common/cross/x86/sources.list.vivid b/eng/common/cross/x86/sources.list.vivid deleted file mode 100644 index 26d37b20fc..0000000000 --- a/eng/common/cross/x86/sources.list.vivid +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ vivid main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ vivid main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ vivid-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ vivid-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ vivid-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ vivid-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ vivid-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ vivid-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.wily b/eng/common/cross/x86/sources.list.wily deleted file mode 100644 index c4b0b442ab..0000000000 --- a/eng/common/cross/x86/sources.list.wily +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ wily main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ wily main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ wily-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ wily-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ wily-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ wily-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ wily-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ wily-security main restricted universe multiverse diff --git a/global.json b/global.json index 5e9039141e..7f8c46a99c 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19256.12" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19257.7" } } From 66eb5f97fa0da1523a56af6f8b779ebc6be7de56 Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Wed, 8 May 2019 11:38:37 -0700 Subject: [PATCH 09/37] add additional shared frameworks --- eng/restore-toolset.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/restore-toolset.ps1 b/eng/restore-toolset.ps1 index 21b799ae50..d1991ddadd 100644 --- a/eng/restore-toolset.ps1 +++ b/eng/restore-toolset.ps1 @@ -17,6 +17,8 @@ function InitializeCustomSDKToolset { $cli = InitializeDotnetCli -install:$true InstallDotNetSharedFramework "1.1.12" + InstallDotNetSharedFramework "2.0.9" + InstallDotNetSharedFramework "2.1.10" } function InstallDotNetSharedFramework([string]$version) { From 875a2cdb9556e31e313d371b1864641aa5c721b6 Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Wed, 8 May 2019 13:20:19 -0700 Subject: [PATCH 10/37] try a separate pool --- .vsts-pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts-pr.yaml b/.vsts-pr.yaml index 9fc4ec94e4..8d449a85a9 100644 --- a/.vsts-pr.yaml +++ b/.vsts-pr.yaml @@ -1,7 +1,7 @@ phases: - phase: Windows - queue: - name: Hosted VS2017 + pool: dotnet-external-vs2019-preview + strategy: timeoutInMinutes: 90 parallel: 6 matrix: From cf1b0d2a78693d897289564d3e84d0a31b42057e Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 9 May 2019 12:43:50 +0000 Subject: [PATCH 11/37] Update dependencies from https://github.com/dotnet/arcade build 20190508.5 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19258.5 --- eng/Version.Details.xml | 4 ++-- eng/common/tools.ps1 | 27 +++++++++++++++++++++++++-- global.json | 2 +- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index bf0138e992..2f5cd4a0aa 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - bda52d7619f9420de46f2c39ffc972864bbcab63 + 9005cee9510d23535c09bd9b6be7c426e526c067 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d86eef1e3a..b0e767e811 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -11,6 +11,12 @@ # Binary log must be enabled on CI. [bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci } + # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). [bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } @@ -442,7 +448,7 @@ function InitializeToolset() { '' | Set-Content $proj - MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile $path = Get-Content $toolsetLocationFile -TotalCount 1 if (!(Test-Path $path)) { @@ -472,6 +478,23 @@ function Stop-Processes() { # Terminates the script if the build fails. # function MSBuild() { + if ($pipelinesLog -and $msbuildEngine) { + $toolsetBuildProject = InitializeToolset + $tf = if ($msbuildEngine -eq "dotnet") { "netcoreapp2.1" } else { "net472" } + $path = Split-Path -parent $toolsetBuildProject + $path = Join-Path $path "$tf\Microsoft.DotNet.Arcade.Sdk.dll" + $args += "/logger:$path" + } + + MSBuild-Core @args +} + +# +# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. +# The arguments are automatically quoted. +# Terminates the script if the build fails. +# +function MSBuild-Core() { if ($ci) { if (!$binaryLog) { throw "Binary log must be enabled in CI build." @@ -549,4 +572,4 @@ if ($ci) { $env:TEMP = $TempDir $env:TMP = $TempDir -} \ No newline at end of file +} diff --git a/global.json b/global.json index 7f8c46a99c..f563177504 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19257.7" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19258.5" } } From f736b7ae209e82d00e4d94a86b88b6b900a35135 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 10 May 2019 12:42:48 +0000 Subject: [PATCH 12/37] Update dependencies from https://github.com/dotnet/arcade build 20190509.9 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19259.9 --- eng/Version.Details.xml | 4 ++-- eng/common/tools.ps1 | 10 +++++----- eng/common/tools.sh | 4 ++-- global.json | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2f5cd4a0aa..6d1401719e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 9005cee9510d23535c09bd9b6be7c426e526c067 + 155080ecb738e189db75f8f2461ebad75d955aa7 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index b0e767e811..9ca177b82a 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -358,7 +358,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild" } + $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -367,7 +367,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = "" } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red ExitWithExitCode 1 @@ -478,11 +478,11 @@ function Stop-Processes() { # Terminates the script if the build fails. # function MSBuild() { - if ($pipelinesLog -and $msbuildEngine) { + if ($pipelinesLog) { + $buildTool = InitializeBuildTool $toolsetBuildProject = InitializeToolset - $tf = if ($msbuildEngine -eq "dotnet") { "netcoreapp2.1" } else { "net472" } $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path "$tf\Microsoft.DotNet.Arcade.Sdk.dll" + $path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") $args += "/logger:$path" } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 9dc565e296..df3eb8bce0 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -165,11 +165,11 @@ function InstallDotNet { local install_script=$_GetDotNetInstallScript local archArg='' - if [[ "$#" -ge "3" ]]; then + if [[ -n "${3:-}" ]]; then archArg="--architecture $3" fi local runtimeArg='' - if [[ "$#" -ge "4" ]]; then + if [[ -n "${4:-}" ]]; then runtimeArg="--runtime $4" fi diff --git a/global.json b/global.json index f563177504..8e3420c4e9 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19258.5" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19259.9" } } From dc496d6f22c67f8fff777ca09ba715fe79c2c2d1 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 11 May 2019 12:34:25 +0000 Subject: [PATCH 13/37] Update dependencies from https://github.com/dotnet/arcade build 20190510.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19260.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6d1401719e..d89f383bc3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 155080ecb738e189db75f8f2461ebad75d955aa7 + 810920cf8a5fc183585304193f8b4e4e189497e9 diff --git a/global.json b/global.json index 8e3420c4e9..0fbb6506d3 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19259.9" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19260.2" } } From 6c13ad5a9778311cb5d93960467854732831592b Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sun, 12 May 2019 12:31:45 +0000 Subject: [PATCH 14/37] Update dependencies from https://github.com/dotnet/arcade build 20190511.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19261.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d89f383bc3..61b1ff0534 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 810920cf8a5fc183585304193f8b4e4e189497e9 + 136dfbfa8a6173dadc107bf7393bdeaceecb4d3a diff --git a/global.json b/global.json index 0fbb6506d3..2c37e56bdb 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19260.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19261.1" } } From deda363dc6e1a36f803bdfe056f5b40ceeaf0a8c Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Mon, 13 May 2019 12:33:14 +0000 Subject: [PATCH 15/37] Update dependencies from https://github.com/dotnet/arcade build 20190512.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19262.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 61b1ff0534..5620d06606 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 136dfbfa8a6173dadc107bf7393bdeaceecb4d3a + 30682cda0dd7ca1765463749dd91ec3cfec75eb9 diff --git a/global.json b/global.json index 2c37e56bdb..c22687a6b3 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19261.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19262.1" } } From 79d3ec5b1c8b3c84f3ed5a56979465f38f8ca71a Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Tue, 14 May 2019 12:33:34 +0000 Subject: [PATCH 16/37] Update dependencies from https://github.com/dotnet/arcade build 20190513.3 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19263.3 --- eng/Version.Details.xml | 4 ++-- eng/common/darc-init.ps1 | 4 ++-- eng/common/darc-init.sh | 4 ++-- global.json | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5620d06606..bd7139dc18 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 30682cda0dd7ca1765463749dd91ec3cfec75eb9 + e6712584bba6e2f0e35a3704793c459ff97c09af diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 81ffd16779..823d72a372 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -19,10 +19,10 @@ function InstallDarcCli ($darcVersion) { # Until we can anonymously query the BAR API for the latest arcade-services # build applied to the PROD channel, this is hardcoded. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19205.4' + $darcVersion = '1.1.0-beta.19258.3' } - $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json' + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index bd7eb46398..36d675738b 100644 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19205.4" +darcVersion="1.1.0-beta.19258.3" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -45,7 +45,7 @@ function InstallDarcCli { echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) fi - local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json" + local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" echo "Installing Darc CLI version $toolset_version..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." diff --git a/global.json b/global.json index c22687a6b3..6c93a925ec 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19262.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19263.3" } } From b136353149246e057abeb3c51f27559558ffd5bd Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Wed, 15 May 2019 12:40:35 +0000 Subject: [PATCH 17/37] Update dependencies from https://github.com/dotnet/arcade build 20190514.13 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19264.13 --- eng/Version.Details.xml | 4 ++-- eng/common/PublishToPackageFeed.proj | 1 + eng/common/cross/arm64/sources.list.buster | 11 +++++++++++ eng/common/cross/arm64/sources.list.stretch | 12 ++++++++++++ eng/common/cross/build-rootfs.sh | 17 +++++++++++++---- eng/common/darc-init.ps1 | 9 +++++---- eng/common/darc-init.sh | 13 +++++++++++-- global.json | 4 ++-- 8 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 eng/common/cross/arm64/sources.list.buster create mode 100644 eng/common/cross/arm64/sources.list.stretch diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index bd7139dc18..dcd15fa1e8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e6712584bba6e2f0e35a3704793c459ff97c09af + 670f6ee1a619a2a7c84cfdfe2a1c84fbe94e1c6b diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index e17f72644e..9120b2d212 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -53,6 +53,7 @@ https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json ]" + echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" @@ -113,12 +113,12 @@ while :; do __LinuxCodeName=trusty fi ;; - xenial) # Ubunry 16.04 + xenial) # Ubuntu 16.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=xenial fi ;; - zesty) # Ununtu 17.04 + zesty) # Ubuntu 17.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=zesty fi @@ -132,7 +132,16 @@ while :; do __LinuxCodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; - # TBD Stretch -> Debian 9, Buster -> Debian 10 + stretch) # Debian 9 + __LinuxCodeName=stretch + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + buster) # Debian 10 + __LinuxCodeName=buster + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; tizen) if [ "$__BuildArch" != "armel" ]; then echo "Tizen is available only for armel." diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 823d72a372..dea7cdd903 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,5 +1,6 @@ param ( - $darcVersion = $null + $darcVersion = $null, + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" ) $verbosity = "m" @@ -16,10 +17,10 @@ function InstallDarcCli ($darcVersion) { Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g" } - # Until we can anonymously query the BAR API for the latest arcade-services - # build applied to the PROD channel, this is hardcoded. + # If the user didn't explicitly specify the darc version, + # query the Maestro API for the correct version of darc to install. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19258.3' + $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 36d675738b..abdd0bc05a 100644 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19258.3" +darcVersion='' +versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -10,6 +11,10 @@ while [[ $# > 0 ]]; do darcVersion=$2 shift ;; + --versionendpoint) + versionEndpoint=$2 + shift + ;; *) echo "Invalid argument: $1" usage @@ -33,6 +38,10 @@ verbosity=m . "$scriptroot/tools.sh" +if [ -z "$darcVersion" ]; then + darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") +fi + function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" @@ -47,7 +56,7 @@ function InstallDarcCli { local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" - echo "Installing Darc CLI version $toolset_version..." + echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) } diff --git a/global.json b/global.json index 6c93a925ec..14fc482c66 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "tools": { - "dotnet": "2.2.203", + "dotnet": "3.0.100-preview5-011568", "vs": { "version": "15.9" }, "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19263.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19264.13" } } From bf2ad3cb30b5024f1c847a8631c37f936d026b89 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 16 May 2019 12:37:24 +0000 Subject: [PATCH 18/37] Update dependencies from https://github.com/dotnet/arcade build 20190516.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19266.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dcd15fa1e8..b538a29268 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 670f6ee1a619a2a7c84cfdfe2a1c84fbe94e1c6b + 37c11672ee11dc2b3365b95a29e0c012f44032be diff --git a/global.json b/global.json index 14fc482c66..400f1e220b 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19264.13" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19266.2" } } From c04bec1495b1a8d9e7e3aebd6707c2ceb8f8c288 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 17 May 2019 12:37:18 +0000 Subject: [PATCH 19/37] Update dependencies from https://github.com/dotnet/arcade build 20190516.4 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19266.4 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b538a29268..5f3afe20ee 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 37c11672ee11dc2b3365b95a29e0c012f44032be + 61cc7d4d347300835925ce5245bfc3ecb6b621d7 diff --git a/global.json b/global.json index 400f1e220b..d081d436b3 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19266.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19266.4" } } From dc9e1c67986369d4f00ec5a06a64bb14c831bdc6 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 18 May 2019 12:38:45 +0000 Subject: [PATCH 20/37] Update dependencies from https://github.com/dotnet/arcade build 20190517.7 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19267.7 --- eng/Version.Details.xml | 4 ++-- eng/common/build.ps1 | 4 ++++ eng/common/dotnet-install.ps1 | 11 ++++++++--- global.json | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5f3afe20ee..1294824b2e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 61cc7d4d347300835925ce5245bfc3ecb6b621d7 + 55ce2900743cb609e51e2c6487e87e42eb627880 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index d7e3799ebd..67046a43f8 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -1,6 +1,7 @@ [CmdletBinding(PositionalBinding=$false)] Param( [string][Alias('c')]$configuration = "Debug", + [string]$platform = $null, [string] $projects, [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, @@ -29,6 +30,7 @@ Param( function Print-Usage() { Write-Host "Common settings:" Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" @@ -77,6 +79,7 @@ function Build { InitializeCustomToolset $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -88,6 +91,7 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` + $platformArg ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 5987943fd6..0b629b8301 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -8,9 +8,14 @@ Param( . $PSScriptRoot\tools.ps1 +$dotnetRoot = Join-Path $RepoRoot ".dotnet" + +$installdir = $dotnetRoot try { - $dotnetRoot = Join-Path $RepoRoot ".dotnet" - InstallDotNet $dotnetRoot $version $architecture $runtime $true + if ($architecture -and $architecture.Trim() -eq "x86") { + $installdir = Join-Path $installdir "x86" + } + InstallDotNet $installdir $version $architecture $runtime $true } catch { Write-Host $_ @@ -19,4 +24,4 @@ catch { ExitWithExitCode 1 } -ExitWithExitCode 0 \ No newline at end of file +ExitWithExitCode 0 diff --git a/global.json b/global.json index d081d436b3..a352ffbf0b 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19266.4" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19267.7" } } From 174db2ccfcbde21ac11723af599fabc1590b2a09 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sun, 19 May 2019 12:36:03 +0000 Subject: [PATCH 21/37] Update dependencies from https://github.com/dotnet/arcade build 20190518.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19268.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1294824b2e..42e8be9e29 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 55ce2900743cb609e51e2c6487e87e42eb627880 + 6d0514d3da59137630b9add5b5e619bce2631f02 diff --git a/global.json b/global.json index a352ffbf0b..f715895656 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19267.7" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19268.2" } } From 80f376cd67a870cc512d4d0c936d26060939f9c4 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Mon, 20 May 2019 12:38:03 +0000 Subject: [PATCH 22/37] Update dependencies from https://github.com/dotnet/arcade build 20190520.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19270.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 42e8be9e29..dadf648d3e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 6d0514d3da59137630b9add5b5e619bce2631f02 + e913fb3b02d4089a91ff91c041c5f6e7c29038b0 diff --git a/global.json b/global.json index f715895656..b75ae7b9cb 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19268.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19270.1" } } From e03f1c215450ddcdb6f2b6915e1b436857756409 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Tue, 21 May 2019 12:37:23 +0000 Subject: [PATCH 23/37] Update dependencies from https://github.com/dotnet/arcade build 20190520.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19270.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dadf648d3e..0d92ab9054 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e913fb3b02d4089a91ff91c041c5f6e7c29038b0 + 3b0760f8789d5d4bd146b36bd9051696d7a56bfa diff --git a/global.json b/global.json index b75ae7b9cb..73c46f08b4 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19270.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19270.2" } } From 3c9a794e3e3ba05deee1d74b8cfedcc7291d3d4d Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Wed, 22 May 2019 12:32:02 +0000 Subject: [PATCH 24/37] Update dependencies from https://github.com/dotnet/arcade build 20190521.7 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19271.7 --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 5 ++ eng/common/tools.ps1 | 6 ++- eng/common/tools.sh | 115 ++++++++++++++++++++++++++++++++++++---- global.json | 2 +- 5 files changed, 119 insertions(+), 13 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0d92ab9054..0f2435a061 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 3b0760f8789d5d4bd146b36bd9051696d7a56bfa + 7a6fb6528b2060cd9e3a3d92535f5b6fdc6b2e82 diff --git a/eng/common/build.sh b/eng/common/build.sh index ce846d888d..6236fc4d38 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -66,6 +66,7 @@ ci=false warn_as_error=true node_reuse=true binary_log=false +pipelines_log=false projects='' configuration='Debug' @@ -92,6 +93,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -pipelineslog|-pl) + pipelines_log=true + ;; -restore|-r) restore=true ;; @@ -146,6 +150,7 @@ while [[ $# > 0 ]]; do done if [[ "$ci" == true ]]; then + pipelines_log=true binary_log=true node_reuse=false fi diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9ca177b82a..9cea610a27 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -213,7 +213,11 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if ($env:VSINSTALLDIR -ne $null) { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { - if ($msbuildCmd.Version -ge $vsMinVersion) { + # Workaround for https://github.com/dotnet/roslyn/issues/35793 + # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) + + if ($msbuildVersion -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index df3eb8bce0..34a23e9476 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -1,8 +1,20 @@ +#!/usr/bin/env bash + # Initialize variables if they aren't already defined. # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +if [[ "$ci" == true ]]; then + pipelines_log=${pipelines_log:-true} +else + pipelines_log=${pipelines_log:-false} +fi + # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} @@ -40,6 +52,78 @@ else use_global_nuget_cache=${use_global_nuget_cache:-true} fi +function EmitError { + if [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + message_type="error" + sourcepath='' + linenumber='' + columnnumber='' + error_code='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-l) + linenumber=$2 + shift + ;; + -columnnumber|-col) + columnnumber=$2 + shift + ;; + -code|-c) + error_code=$2 + shift + ;; + *) + break + ;; + esac + + shift + done + + message='##vso[task.logissue' + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + else + message="$message;sourcepath=${BASH_SOURCE[1]}" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + else + message="$message;linenumber=${BASH_LINENO[0]}" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + + echo "$message" +} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -65,7 +149,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - echo "Error: Cannot find \"$key\" in $global_json_file" >&2 + EmitError "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -126,7 +210,7 @@ function InitializeDotNetCli { if [[ "$install" == true ]]; then InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" else - echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2 + EmitError "Unable to find dotnet with SDK version '$dotnet_sdk_version'" ExitWithExitCode 1 fi fi @@ -179,7 +263,7 @@ function InstallDotNet { fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 + EmitError "Failed to install dotnet SDK (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -216,6 +300,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" + _InitializeBuildToolFramework="netcoreapp2.1" } function GetNuGetPackageCachePath { @@ -264,7 +349,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - echo "Toolset version $toolsetVersion has not been restored." >&2 + EmitError "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 2 fi @@ -276,12 +361,12 @@ function InitializeToolset { fi echo '' > "$proj" - MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - echo "Invalid toolset path: $toolset_build_proj" >&2 + EmitError "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -304,14 +389,26 @@ function StopProcesses { } function MSBuild { + args=$@ + if [[ "$pipelines_log" == true ]]; then + InitializeBuildTool + InitializeToolset + _toolset_dir="${_InitializeToolset%/*}" + _loggerPath="$_toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + args=( "${args[@]}" "-logger:$_loggerPath" ) + fi + MSBuild-Core ${args[@]} +} + +function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true ]]; then - echo "Binary log must be enabled in CI build." >&2 + EmitError "Binary log must be enabled in CI build." ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - echo "Node reuse must be disabled in CI build." >&2 + EmitError "Node reuse must be disabled in CI build." ExitWithExitCode 1 fi fi @@ -325,7 +422,7 @@ function MSBuild { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? - echo "Build failed (exit code '$exit_code')." >&2 + EmitError "Build failed (exit code '$exit_code')." ExitWithExitCode $exit_code } } diff --git a/global.json b/global.json index 73c46f08b4..5bbb9fd50d 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19270.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19271.7" } } From 4afd0b9943d096149edf1b221b6d56816348e8e8 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 23 May 2019 12:43:47 +0000 Subject: [PATCH 25/37] Update dependencies from https://github.com/dotnet/arcade build 20190522.13 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19272.13 --- eng/Version.Details.xml | 4 ++-- eng/common/templates/steps/send-to-helix.yml | 3 +++ global.json | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0f2435a061..1d212db267 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 7a6fb6528b2060cd9e3a3d92535f5b6fdc6b2e82 + 86e674361bdcefecbd8199ab62d0b1a6cb25703d diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index d1ce577db5..05df886f55 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,6 +5,7 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects @@ -35,6 +36,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} @@ -64,6 +66,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} diff --git a/global.json b/global.json index 5bbb9fd50d..3f84a1458b 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19271.7" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19272.13" } } From 6c1d2aa45c1c457246f7d6cd0ca9ba83a6f8e3c2 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 24 May 2019 12:43:37 +0000 Subject: [PATCH 26/37] Update dependencies from https://github.com/dotnet/arcade build 20190523.11 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19273.11 --- eng/Version.Details.xml | 4 ++-- eng/common/tools.sh | 9 +++++---- global.json | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1d212db267..a232d2aff6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 86e674361bdcefecbd8199ab62d0b1a6cb25703d + 02a90cc69d2d32bbde9e6e0e41186711c591de27 diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 34a23e9476..fd26f6fbb2 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -389,14 +389,15 @@ function StopProcesses { } function MSBuild { - args=$@ + local args=$@ if [[ "$pipelines_log" == true ]]; then InitializeBuildTool InitializeToolset - _toolset_dir="${_InitializeToolset%/*}" - _loggerPath="$_toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" - args=( "${args[@]}" "-logger:$_loggerPath" ) + local toolset_dir="${_InitializeToolset%/*}" + local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + args=( "${args[@]}" "-logger:$logger_path" ) fi + MSBuild-Core ${args[@]} } diff --git a/global.json b/global.json index 3f84a1458b..173483c857 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19272.13" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19273.11" } } From e7fb735561c732207cafd513a4c259af211a0a4b Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 25 May 2019 12:41:50 +0000 Subject: [PATCH 27/37] Update dependencies from https://github.com/dotnet/arcade build 20190524.6 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19274.6 --- eng/Version.Details.xml | 4 ++-- eng/common/PublishToPackageFeed.proj | 1 + global.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a232d2aff6..a17980fc6e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 02a90cc69d2d32bbde9e6e0e41186711c591de27 + b5016f5688dc8ca9f3e4811ee7e2e86ad8907a40 diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index 9120b2d212..a1b1333723 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -54,6 +54,7 @@ https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json Date: Wed, 29 May 2019 12:46:01 +0000 Subject: [PATCH 28/37] Update dependencies from https://github.com/dotnet/arcade build 20190528.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19278.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a17980fc6e..0bbf9d26f4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - b5016f5688dc8ca9f3e4811ee7e2e86ad8907a40 + 11f90a2a260422201895de58e57170131ab4efe7 diff --git a/global.json b/global.json index 186fa44395..afc50019f4 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19274.6" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19278.1" } } From bcda5eefc8960ce91dde13cb28860318abad7196 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 30 May 2019 12:44:47 +0000 Subject: [PATCH 29/37] Update dependencies from https://github.com/dotnet/arcade build 20190529.5 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19279.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0bbf9d26f4..e15525992b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 11f90a2a260422201895de58e57170131ab4efe7 + fb62c6377a6bd163af2a7516260f064498942585 diff --git a/global.json b/global.json index afc50019f4..563344a6c3 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19278.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19279.5" } } From 766ef53e277dbaf2e45b7c5881d118d675e96aa0 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 31 May 2019 12:40:06 +0000 Subject: [PATCH 30/37] Update dependencies from https://github.com/dotnet/arcade build 20190530.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19280.2 --- eng/Version.Details.xml | 4 +- eng/common/LoggingCommandFunctions.ps1 | 146 +++++++++++++++++++++++++ eng/common/build.ps1 | 3 +- eng/common/tools.ps1 | 103 +++++++++++++---- global.json | 2 +- 5 files changed, 231 insertions(+), 27 deletions(-) create mode 100644 eng/common/LoggingCommandFunctions.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e15525992b..c1d7f0416b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - fb62c6377a6bd163af2a7516260f064498942585 + 7c50d548001a83a18449ad4dda370122ede5fbf6 diff --git a/eng/common/LoggingCommandFunctions.ps1 b/eng/common/LoggingCommandFunctions.ps1 new file mode 100644 index 0000000000..c225eaecbf --- /dev/null +++ b/eng/common/LoggingCommandFunctions.ps1 @@ -0,0 +1,146 @@ +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 + +# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 + +$script:loggingCommandPrefix = '##vso[' +$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? + New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } + New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } + New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } + New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } +) +# TODO: BUG: Escape % ??? +# TODO: Add test to verify don't need to escape "=". + +<######################################## +# Private functions. +########################################> +function Format-LoggingCommandData { + [CmdletBinding()] + param([string]$Value, [switch]$Reverse) + + if (!$Value) { + return '' + } + + if (!$Reverse) { + foreach ($mapping in $script:loggingCommandEscapeMappings) { + $Value = $Value.Replace($mapping.Token, $mapping.Replacement) + } + } else { + for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { + $mapping = $script:loggingCommandEscapeMappings[$i] + $Value = $Value.Replace($mapping.Replacement, $mapping.Token) + } + } + + return $Value +} + +function Format-LoggingCommand { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Area, + [Parameter(Mandatory = $true)] + [string]$Event, + [string]$Data, + [hashtable]$Properties) + + # Append the preamble. + [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder + $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) + + # Append the properties. + if ($Properties) { + $first = $true + foreach ($key in $Properties.Keys) { + [string]$value = Format-LoggingCommandData $Properties[$key] + if ($value) { + if ($first) { + $null = $sb.Append(' ') + $first = $false + } else { + $null = $sb.Append(';') + } + + $null = $sb.Append("$key=$value") + } + } + } + + # Append the tail and output the value. + $Data = Format-LoggingCommandData $Data + $sb.Append(']').Append($Data).ToString() +} + +function Write-LoggingCommand { + [CmdletBinding(DefaultParameterSetName = 'Parameters')] + param( + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Area, + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Event, + [Parameter(ParameterSetName = 'Parameters')] + [string]$Data, + [Parameter(ParameterSetName = 'Parameters')] + [hashtable]$Properties, + [Parameter(Mandatory = $true, ParameterSetName = 'Object')] + $Command, + [switch]$AsOutput) + + if ($PSCmdlet.ParameterSetName -eq 'Object') { + Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput + return + } + + $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties + if ($AsOutput) { + $command + } else { + Write-Host $command + } +} + +function Write-LogIssue { + [CmdletBinding()] + param( + [ValidateSet('warning', 'error')] + [Parameter(Mandatory = $true)] + [string]$Type, + [string]$Message, + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } + if ($AsOutput) { + return $command + } + + if ($Type -eq 'error') { + $foregroundColor = $host.PrivateData.ErrorForegroundColor + $backgroundColor = $host.PrivateData.ErrorBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Red + $backgroundColor = [System.ConsoleColor]::Black + } + } else { + $foregroundColor = $host.PrivateData.WarningForegroundColor + $backgroundColor = $host.PrivateData.WarningBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Yellow + $backgroundColor = [System.ConsoleColor]::Black + } + } + + Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor +} \ No newline at end of file diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 67046a43f8..4cb2ce489b 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -133,9 +133,8 @@ try { Build } catch { - Write-Host $_ - Write-Host $_.Exception Write-Host $_.ScriptStackTrace + Write-PipelineTaskError -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9cea610a27..3983d719be 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -92,6 +92,68 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + if(!$ci) { + if($Type -eq 'error') { + Write-Error $Message + return + } + elseif ($Type -eq 'warning') { + Write-Warning $Message + return + } + } + + if(($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + if(-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters +} + +function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput) + + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'issecret' = $Secret + } -AsOutput:$AsOutput + } +} + +function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string]$Path, + [switch]$AsOutput) + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } +} + function InitializeDotNetCli([bool]$install) { if (Test-Path variable:global:_DotNetInstallDir) { return $global:_DotNetInstallDir @@ -134,7 +196,7 @@ function InitializeDotNetCli([bool]$install) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { - Write-Host "Unable to find dotnet with SDK version '$dotnetSdkVersion'" -ForegroundColor Red + Write-PipelineTaskError "Unable to find dotnet with SDK version '$dotnetSdkVersion'" ExitWithExitCode 1 } } @@ -147,12 +209,10 @@ function InitializeDotNetCli([bool]$install) { # It also ensures that VS msbuild will use the downloaded sdk targets. $env:PATH = "$dotnetRoot;$env:PATH" - if ($ci) { - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build - Write-Host "##vso[task.prependpath]$dotnetRoot" - Write-Host "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" - Write-Host "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" - } + # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + Write-PipelinePrependPath -Path $dotnetRoot + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -184,7 +244,7 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit & $installScript @installParameters if ($lastExitCode -ne 0) { - Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red + Write-PipelineTaskError -Message "Failed to install dotnet cli (exit code '$lastExitCode')." ExitWithExitCode $lastExitCode } } @@ -358,7 +418,7 @@ function InitializeBuildTool() { if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { - Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red + Write-PipelineTaskError "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } @@ -367,13 +427,13 @@ function InitializeBuildTool() { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore } catch { - Write-Host $_ -ForegroundColor Red + Write-PipelineTaskError $_ ExitWithExitCode 1 } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { - Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red + Write-PipelineTaskError "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 } @@ -390,7 +450,7 @@ function GetDefaultMSBuildEngine() { return "dotnet" } - Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red + Write-PipelineTaskError "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." ExitWithExitCode 1 } @@ -441,7 +501,7 @@ function InitializeToolset() { } if (-not $restore) { - Write-Host "Toolset version $toolsetVersion has not been restored." -ForegroundColor Red + Write-PipelineTaskError "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 1 } @@ -526,7 +586,7 @@ function MSBuild-Core() { $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-Host "Build failed." -ForegroundColor Red + Write-PipelineTaskError "Build failed." $buildLog = GetMSBuildBinaryLogCommandLineArgument $args if ($buildLog -ne $null) { @@ -554,6 +614,8 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +. $PSScriptRoot\LoggingCommandFunctions.ps1 + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $ArtifactsDir = Join-Path $RepoRoot "artifacts" @@ -569,11 +631,8 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir -if ($ci) { - Write-Host "##vso[task.setvariable variable=Artifacts]$ArtifactsDir" - Write-Host "##vso[task.setvariable variable=Artifacts.Toolset]$ToolsetDir" - Write-Host "##vso[task.setvariable variable=Artifacts.Log]$LogDir" - - $env:TEMP = $TempDir - $env:TMP = $TempDir -} +Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir +Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir +Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir +Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir +Write-PipelineSetVariable -Name 'TMP' -Value $TempDir diff --git a/global.json b/global.json index 563344a6c3..52e1d0932e 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19279.5" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19280.2" } } From cf27d644cc037de0426827354f5205e33ab498d1 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 1 Jun 2019 12:30:57 +0000 Subject: [PATCH 31/37] Update dependencies from https://github.com/dotnet/arcade build 20190531.5 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19281.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c1d7f0416b..c991366c2c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 7c50d548001a83a18449ad4dda370122ede5fbf6 + c901850aa1e175c124da771863dacc763743a520 diff --git a/global.json b/global.json index 52e1d0932e..8e1c8d559e 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19280.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19281.5" } } From 7043644aa22d87e02c5cdb68a7cc59a0a518955a Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Sun, 2 Jun 2019 12:50:26 -0700 Subject: [PATCH 32/37] use 2017 pool --- .vsts-pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts-pr.yaml b/.vsts-pr.yaml index 8d449a85a9..9fc4ec94e4 100644 --- a/.vsts-pr.yaml +++ b/.vsts-pr.yaml @@ -1,7 +1,7 @@ phases: - phase: Windows - pool: dotnet-external-vs2019-preview - strategy: + queue: + name: Hosted VS2017 timeoutInMinutes: 90 parallel: 6 matrix: From e31fc878e8ec8b019f6366de895dba5bb1f33f2b Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Sun, 2 Jun 2019 12:54:01 -0700 Subject: [PATCH 33/37] install 2.2 sdk --- eng/restore-toolset.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/restore-toolset.ps1 b/eng/restore-toolset.ps1 index d1991ddadd..69e5071805 100644 --- a/eng/restore-toolset.ps1 +++ b/eng/restore-toolset.ps1 @@ -19,6 +19,7 @@ function InitializeCustomSDKToolset { InstallDotNetSharedFramework "1.1.12" InstallDotNetSharedFramework "2.0.9" InstallDotNetSharedFramework "2.1.10" + InstallDotNetSharedFramework "2.2.5" } function InstallDotNetSharedFramework([string]$version) { From 519b83baaec0d318450f908b9a6b697052ddd65d Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sun, 2 Jun 2019 19:54:24 +0000 Subject: [PATCH 34/37] Update dependencies from https://github.com/dotnet/arcade build 20190601.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19301.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c991366c2c..ad46b7bd74 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - c901850aa1e175c124da771863dacc763743a520 + 8fd0b50b9d689e8d87ab202f11b7c9fbe81271c4 diff --git a/global.json b/global.json index 8e1c8d559e..16b4e4f67c 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19281.5" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19301.2" } } From 71fff3bcbc7ad2bea65a4a1d824617f8ccfed143 Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Sun, 2 Jun 2019 13:01:23 -0700 Subject: [PATCH 35/37] use new dnceng pool --- .vsts-pr.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.vsts-pr.yaml b/.vsts-pr.yaml index 9fc4ec94e4..eddb0e0ef2 100644 --- a/.vsts-pr.yaml +++ b/.vsts-pr.yaml @@ -1,9 +1,10 @@ -phases: -- phase: Windows - queue: - name: Hosted VS2017 - timeoutInMinutes: 90 - parallel: 6 +jobs: +- job: Windows + pool: + name: NetCorePublic-Pool + queue: buildpool.windows.10.amd64.vs2019.pre.open + strategy: + maxParallel: 6 matrix: Samples Debug Test: _args: -test @@ -29,6 +30,8 @@ phases: _args: -sign _configuration: Release _solution: Roslyn-SDK + timeoutInMinutes: 90 + steps: - script: eng\PRBuild.cmd $(_args) -configuration $(_configuration) -prepareMachine -projects $(Build.SourcesDirectory)\$(_solution).sln /p:OfficialBuild=false - task: PublishBuildArtifacts@1 From 0d5f06d3a71c8347aca4eac6e88570f15e49fbf9 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Mon, 3 Jun 2019 12:32:32 +0000 Subject: [PATCH 36/37] Update dependencies from https://github.com/dotnet/arcade build 20190602.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19302.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ad46b7bd74..b8adc39c26 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 8fd0b50b9d689e8d87ab202f11b7c9fbe81271c4 + e6a5d5f970bb872451c6310ae34eda31041fb552 diff --git a/global.json b/global.json index 16b4e4f67c..7e10eb112d 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19301.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19302.2" } } From 19163f831503770e5c285b471e3feb2f2134ed40 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Tue, 4 Jun 2019 12:43:57 +0000 Subject: [PATCH 37/37] Update dependencies from https://github.com/dotnet/arcade build 20190604.1 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19304.1 --- eng/Version.Details.xml | 4 +- eng/common/sdl/NuGet.config | 13 ++++ eng/common/sdl/execute-all-sdl-tools.ps1 | 97 ++++++++++++++++++++++++ eng/common/sdl/init-sdl.ps1 | 48 ++++++++++++ eng/common/sdl/packages.config | 4 + eng/common/sdl/push-gdn.ps1 | 51 +++++++++++++ eng/common/sdl/run-sdl.ps1 | 65 ++++++++++++++++ global.json | 2 +- 8 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 eng/common/sdl/NuGet.config create mode 100644 eng/common/sdl/execute-all-sdl-tools.ps1 create mode 100644 eng/common/sdl/init-sdl.ps1 create mode 100644 eng/common/sdl/packages.config create mode 100644 eng/common/sdl/push-gdn.ps1 create mode 100644 eng/common/sdl/run-sdl.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b8adc39c26..cf9c04d6eb 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e6a5d5f970bb872451c6310ae34eda31041fb552 + d2599acd9703ef747dfb4329ec3e3beff182e755 diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config new file mode 100644 index 0000000000..0c5451c114 --- /dev/null +++ b/eng/common/sdl/NuGet.config @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 new file mode 100644 index 0000000000..74080f22d1 --- /dev/null +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -0,0 +1,97 @@ +Param( + [string] $GuardianPackageName, # Required: the name of guardian CLI pacakge (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName="master", # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory, # Required: the directory where source files are located + [string] $ArtifactsDirectory, # Required: the directory where build artifacts are located + [string] $DncEngAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCHNAME, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +#Replace repo names to the format of org/repo +if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; +} +else{ + $RepoName = $Repository; +} + +if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) +} else { + $guardianCliLocation = $GuardianCliLocation +} + +$ValidPath = Test-Path $guardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -DncEngAccessToken $DncEngAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" + +if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." + exit 1 + } +} + +if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} +if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -DncEngAccessToken $DncEngAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} + +if ($UpdateBaseline) { + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Update baseline" +} + +if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." + exit 1 + } +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 new file mode 100644 index 0000000000..cbf5c36a8f --- /dev/null +++ b/eng/common/sdl/init-sdl.ps1 @@ -0,0 +1,48 @@ +Param( + [string] $GuardianCliLocation, + [string] $Repository, + [string] $BranchName="master", + [string] $WorkingDirectory, + [string] $DncEngAccessToken, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$DncEngAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$zipFile = "$WorkingDirectory/gdn.zip" + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory ".gdn") +Try +{ + # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead + Write-Host "Downloading gdn folder from internal config repostiory..." + Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile + if (Test-Path $gdnFolder) { + # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) + Remove-Item -Force -Recurse $gdnFolder + } + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) + Write-Host $gdnFolder +} Catch [System.Net.WebException] { + # if the folder does not exist, we'll do a guardian init and push it to the remote repository + Write-Host "Initializing Guardian..." + Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" + & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian init failed with exit code $LASTEXITCODE." + } + # We create the mainbaseline so it can be edited later + Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" + & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." + } + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -DncEngAccessToken $DncEngAccessToken -PushReason "Initialize gdn folder" +} \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config new file mode 100644 index 0000000000..b054737df1 --- /dev/null +++ b/eng/common/sdl/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 new file mode 100644 index 0000000000..cacaf8e912 --- /dev/null +++ b/eng/common/sdl/push-gdn.ps1 @@ -0,0 +1,51 @@ +Param( + [string] $Repository, + [string] $BranchName="master", + [string] $GdnFolder, + [string] $DncEngAccessToken, + [string] $PushReason +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We create the temp directory where we'll store the sdl-config repository +$sdlDir = Join-Path $env:TEMP "sdl" +if (Test-Path $sdlDir) { + Remove-Item -Force -Recurse $sdlDir +} + +Write-Host "git clone https://dnceng:`$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$DncEngAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +if ($LASTEXITCODE -ne 0) { + Write-Error "Git clone failed with exit code $LASTEXITCODE." +} +# We copy the .gdn folder from our local run into the git repository so it can be committed +$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" +if (Get-Command Robocopy) { + Robocopy /S $GdnFolder $sdlRepositoryFolder +} else { + rsync -r $GdnFolder $sdlRepositoryFolder +} +# cd to the sdl-config directory so we can run git there +Push-Location $sdlDir +# git add . --> git commit --> git push +Write-Host "git add ." +git add . +if ($LASTEXITCODE -ne 0) { + Write-Error "Git add failed with exit code $LASTEXITCODE." +} +Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" +git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." +} +Write-Host "git push" +git push +if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." +} + +# Return to the original directory +Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 new file mode 100644 index 0000000000..e6a86d03a2 --- /dev/null +++ b/eng/common/sdl/run-sdl.ps1 @@ -0,0 +1,65 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + [string[]] $ToolsList, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We store config files in the r directory of .gdn +Write-Host $ToolsList +$gdnConfigPath = Join-Path $GdnFolder "r" +$ValidPath = Test-Path $GuardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +foreach ($tool in $ToolsList) { + $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" + $config = $False + Write-Host $tool + # We have to manually configure tools that run on source to look at the source directory only + if ($tool -eq "credscan") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + if ($tool -eq "policheck") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + + Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" + if ($config) { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } +} + diff --git a/global.json b/global.json index 7e10eb112d..cb6836f1b0 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19302.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19304.1" } }