From ad8b4991ee7f0636d7bf3e820dd18f3867db3c23 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 11 Jul 2019 12:48:07 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20190710.8 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19360.8 --- eng/Version.Details.xml | 4 +- eng/common/SigningValidation.proj | 2 +- eng/common/performance/perfhelixpublish.proj | 77 ++++++++ eng/common/performance/performance-setup.ps1 | 91 +++++++++ eng/common/performance/performance-setup.sh | 176 ++++++++++++++++++ eng/common/pipeline-logging-functions.ps1 | 7 +- .../post-build/trigger-subscriptions.ps1 | 6 +- eng/common/templates/job/performance.yml | 93 +++++++++ .../channels/internal-servicing.yml | 8 +- .../channels/public-dev-release.yml | 1 + .../post-build/channels/public-release.yml | 8 +- .../channels/public-validation-release.yml | 1 + .../templates/post-build/common-variables.yml | 3 + .../templates/steps/perf-send-to-helix.yml | 66 +++++++ eng/common/tools.ps1 | 4 +- global.json | 2 +- 16 files changed, 529 insertions(+), 20 deletions(-) create mode 100644 eng/common/performance/perfhelixpublish.proj create mode 100644 eng/common/performance/performance-setup.ps1 create mode 100644 eng/common/performance/performance-setup.sh create mode 100644 eng/common/templates/job/performance.yml create mode 100644 eng/common/templates/steps/perf-send-to-helix.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0c5078f9f9c..bfb72eca155 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 0f5dd7680174620f31c9a00cdb2ac0b0e70e631f + a6ae1b637ed236354529992729af875f6c8a180a diff --git a/eng/common/SigningValidation.proj b/eng/common/SigningValidation.proj index 7045fb6fb9d..3d0ac80af3f 100644 --- a/eng/common/SigningValidation.proj +++ b/eng/common/SigningValidation.proj @@ -3,7 +3,7 @@ + + + $(WorkItemDirectory) + $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + 4:00 + + + + + $(WorkItemDirectory) + $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)" + 4:00 + + + \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 new file mode 100644 index 00000000000..7e5441f7974 --- /dev/null +++ b/eng/common/performance/performance-setup.ps1 @@ -0,0 +1,91 @@ +Param( + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, + [string] $CoreRootDirectory, + [string] $Architecture="x64", + [string] $Framework="netcoreapp3.0", + [string] $CompilationMode="Tiered", + [string] $Repository=$env:BUILD_REPOSITORY_NAME, + [string] $Branch=$env:BUILD_SOURCEBRANCH, + [string] $CommitSha=$env:BUILD_SOURCEVERSION, + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, + [string] $RunCategories="coreclr corefx", + [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", + [string] $Kind="micro", + [switch] $Internal, + [string] $Configurations="CompilationMode=$CompilationMode" +) + +$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") +$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) + +$PayloadDirectory = (Join-Path $SourceDirectory "Payload") +$PerformanceDirectory = (Join-Path $PayloadDirectory "performance") +$WorkItemDirectory = (Join-Path $SourceDirectory "workitem") +$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" +$Creator = $env:BUILD_DEFINITIONNAME +$PerfLabArguments = "" +$HelixSourcePrefix = "pr" + +$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" + +if ($Framework.StartsWith("netcoreapp")) { + $Queue = "Windows.10.Amd64.ClientRS4.Open" +} + +if ($Internal) { + $Queue = "Windows.10.Amd64.ClientRS5.Perf" + $PerfLabArguments = "--upload-to-perflab-container" + $ExtraBenchmarkDotNetArguments = "" + $Creator = "" + $HelixSourcePrefix = "official" +} + +$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations" +$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" + +if ($RunFromPerformanceRepo) { + $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" + + robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git +} +else { + git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory +} + +if ($UseCoreRun) { + $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") + Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot +} + +$DocsDir = (Join-Path $PerformanceDirectory "docs") +robocopy $DocsDir $WorkItemDirectory + +# Set variables that we will need to have in future steps +$ci = $true + +. "$PSScriptRoot\..\pipeline-logging-functions.ps1" + +# Directories +Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false + +# Script Arguments +Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false + +# Helix Arguments +Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false + +exit 0 \ No newline at end of file diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh new file mode 100644 index 00000000000..126da5f76d4 --- /dev/null +++ b/eng/common/performance/performance-setup.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash + +source_directory=$BUILD_SOURCESDIRECTORY +core_root_directory= +architecture=x64 +framework=netcoreapp3.0 +compilation_mode=tiered +repository=$BUILD_REPOSITORY_NAME +branch=$BUILD_SOURCEBRANCH +commit_sha=$BUILD_SOURCEVERSION +build_number=$BUILD_BUILDNUMBER +internal=false +kind="micro" +run_categories="coreclr corefx" +csproj="src\benchmarks\micro\MicroBenchmarks.csproj" +configurations= +run_from_perf_repo=false +use_core_run=true + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --sourcedirectory) + source_directory=$2 + shift 2 + ;; + --corerootdirectory) + core_root_directory=$2 + shift 2 + ;; + --architecture) + architecture=$2 + shift 2 + ;; + --framework) + framework=$2 + shift 2 + ;; + --compilationmode) + compilation_mode=$2 + shift 2 + ;; + --repository) + repository=$2 + shift 2 + ;; + --branch) + branch=$2 + shift 2 + ;; + --commitsha) + commit_sha=$2 + shift 2 + ;; + --buildnumber) + build_number=$2 + shift 2 + ;; + --kind) + kind=$2 + shift 2 + ;; + --runcategories) + run_categories=$2 + shift 2 + ;; + --csproj) + csproj=$2 + shift 2 + ;; + --internal) + internal=true + shift 1 + ;; + --configurations) + configurations=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" + echo " --architecture Architecture of the testing being run" + echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure." + echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\"" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --framework The framework to run, if not running in master" + echo " --compliationmode The compilation mode if not passing --configurations" + echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY" + echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME" + echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH" + echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION" + echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER" + echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj" + echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" + echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" + echo " --internal If the benchmarks are running as an official job." + echo "" + exit 0 + ;; + esac +done + +if [[ "$repository" == "dotnet/performance" ]]; then + run_from_perf_repo=true +fi + +if [ -z "$configurations" ]; then + configurations="CompliationMode=$compilation_mode" +fi + +if [ -z "$core_root_directory" ]; then + use_core_run=false +fi + +payload_directory=$source_directory/Payload +performance_directory=$payload_directory/performance +workitem_directory=$source_directory/workitem +extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" +perflab_arguments= +queue=Ubuntu.1804.Amd64.Open +creator=$BUILD_DEFINITIONNAME +helix_source_prefix="pr" + +if [[ "$internal" == true ]]; then + perflab_arguments="--upload-to-perflab-container" + helix_source_prefix="official" + creator= + extra_benchmark_dotnet_arguments= + + if [[ "$architecture" = "arm64" ]]; then + queue=Ubuntu.1804.Arm64.Perf + else + queue=Ubuntu.1804.Amd64.Perf + fi +fi + +common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations" +setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" + +if [[ "$run_from_perf_repo" = true ]]; then + payload_directory= + workitem_directory=$source_directory + performance_directory=$workitem_directory + setup_arguments="--perf-hash $commit_sha $common_setup_arguments" +else + git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory + + docs_directory=$performance_directory/docs + mv $docs_directory $workitem_directory +fi + +if [[ "$use_core_run" = true ]]; then + new_core_root=$payload_directory/Core_Root + mv $core_root_directory $new_core_root +fi + +# Make sure all of our variables are available for future steps +echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run" +echo "##vso[task.setvariable variable=Architecture]$architecture" +echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory" +echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory" +echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory" +echo "##vso[task.setvariable variable=Queue]$queue" +echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments" +echo "##vso[task.setvariable variable=Python]python3" +echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments" +echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments" +echo "##vso[task.setvariable variable=BDNCategories]$run_categories" +echo "##vso[task.setvariable variable=TargetCsproj]$csproj" +echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo" +echo "##vso[task.setvariable variable=Creator]$creator" +echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix" +echo "##vso[task.setvariable variable=Kind]$kind" +echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework" \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 7b61376f8aa..af5f48aaceb 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -77,13 +77,14 @@ function Write-PipelineTaskError { [string]$Name, [string]$Value, [switch]$Secret, - [switch]$AsOutput) - + [switch]$AsOutput, + [bool]$IsMultiJobVariable=$true) + if($ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ 'variable' = $Name 'isSecret' = $Secret - 'isOutput' = 'true' + 'isOutput' = $IsMultiJobVariable } -AsOutput:$AsOutput } } diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index db8a839457a..1a91dab0371 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -19,14 +19,14 @@ function Get-Headers([string]$accept, [string]$barToken) { } # Get all the $SourceRepo subscriptions -$normalizedSurceRepo = $SourceRepo.Replace('dnceng@', '') -$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSurceRepo&api-version=$apiVersion" +$normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') +$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSourceRepo&api-version=$apiVersion" $headers = Get-Headers 'application/json' $barToken $subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json if (!$subscriptions) { - Write-Host "No subscriptions found for source repo '$normalizedSurceRepo' in channel '$ChannelId'" + Write-Host "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" return } diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml new file mode 100644 index 00000000000..ef809253d1a --- /dev/null +++ b/eng/common/templates/job/performance.yml @@ -0,0 +1,93 @@ +parameters: + steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo) + variables: [] # optional -- list of additional variables to send to the template + jobName: '' # required -- job name + displayName: '' # optional -- display name for the job. Will use jobName if not passed + pool: '' # required -- name of the Build pool + container: '' # required -- name of the container + extraSetupParameters: '' # optional -- extra arguments to pass to the setup script + frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against + continueOnError: 'false' # optional -- determines whether to continue the build if the step errors + dependsOn: '' # optional -- dependencies of the job + timeoutInMinutes: 320 # optional -- timeout for the job + enableTelemetry: false # optional -- enable for telemetry + +jobs: +- template: ../jobs/jobs.yml + parameters: + dependsOn: ${{ parameters.dependsOn }} + enableTelemetry: ${{ parameters.enableTelemetry }} + enablePublishBuildArtifacts: true + continueOnError: ${{ parameters.continueOnError }} + + jobs: + - job: '${{ parameters.jobName }}' + + ${{ if ne(parameters.displayName, '') }}: + displayName: '${{ parameters.displayName }}' + ${{ if eq(parameters.displayName, '') }}: + displayName: '${{ parameters.jobName }}' + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + + - ${{ each variable in parameters.variables }}: + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + - IsInternal: '' + - HelixApiAccessToken: '' + - HelixPreCommand: '' + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(variables['Agent.Os'], 'Windows_NT') }}: + - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' + - IsInternal: -Internal + - ${{ if ne(variables['Agent.Os'], 'Windows_NT') }}: + - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' + - IsInternal: --internal + - group: DotNet-HelixApi-Access + - group: dotnet-benchview + + workspace: + clean: all + pool: + ${{ parameters.pool }} + container: ${{ parameters.container }} + strategy: + matrix: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + steps: + - checkout: self + clean: true + # Run all of the steps to setup repo + - ${{ each step in parameters.steps }}: + - ${{ step }} + - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }} + displayName: Performance Setup (Windows) + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }} + displayName: Performance Setup (Unix) + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) + displayName: Run ci setup script + # Run perf testing in helix + - template: /eng/common/templates/steps/perf-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + Creator: $(Creator) + WorkItemTimeout: 4:00 # 4 hours + WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy + CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/internal-servicing.yml index 808d46b17f2..50ad724fc0c 100644 --- a/eng/common/templates/post-build/channels/internal-servicing.yml +++ b/eng/common/templates/post-build/channels/internal-servicing.yml @@ -81,10 +81,10 @@ stages: /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) + /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) + /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) + /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' /p:BuildAssetRegistryToken='$(MaestroAccessToken)' @@ -167,4 +167,4 @@ stages: - template: ../promote-build.yml parameters: - ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} \ No newline at end of file + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index 79c6822db72..bdc631016b6 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -77,6 +77,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicDevRelease_30_Channel_Id) + /p:ArtifactsCategory=.NetCore /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/public-release.yml index 25923020df8..574cb1c2b92 100644 --- a/eng/common/templates/post-build/channels/public-release.yml +++ b/eng/common/templates/post-build/channels/public-release.yml @@ -81,10 +81,10 @@ stages: /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)/Nuget/NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) + /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) + /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe /p:BARBuildId=$(BARBuildId) /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' /p:BuildAssetRegistryToken='$(MaestroAccessToken)' diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 114477d3adb..f12f402ad9a 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -48,6 +48,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + /p:ArtifactsCategory=.NetCoreValidation /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 82834673520..42df4ae77e3 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -14,5 +14,8 @@ variables: # Whether the build is internal or not IsInternalBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + # Storage account name for proxy-backed feeds + ProxyBackedFeedsAccountName: dotnetfeed + SourceLinkCLIVersion: 3.0.0 SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml new file mode 100644 index 00000000000..b3ea9acf1f1 --- /dev/null +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -0,0 +1,66 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\perfhelixpublish.proj /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/perfhelixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60741f03901..9abaac015fb 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -84,7 +84,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { return $global:LASTEXITCODE = $process.ExitCode } finally { - # If we didn't finish then an error occured or the user hit ctrl-c. Either + # If we didn't finish then an error occurred or the user hit ctrl-c. Either # way kill the process if (-not $finished) { $process.Kill() @@ -147,7 +147,7 @@ function InitializeDotNetCli([bool]$install) { # It also ensures that VS msbuild will use the downloaded sdk targets. $env:PATH = "$dotnetRoot;$env:PATH" - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' diff --git a/global.json b/global.json index 4c66387e58e..60dbe5efe92 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19359.6", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19360.8", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } }