From 8ee2bbab3decc027b35573b7c8542dc92d85ea80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 19 Feb 2019 13:13:50 -0800 Subject: [PATCH] Read IBC and VS bootstrapper info from PublishData.json (#33410) * Read IBC and VS bootstrapper info from PublishData.json * Fix IBC directory path --- azure-pipelines-official.yml | 10 ++--- eng/build-utils.ps1 | 32 ++++++++++++++++ eng/build.ps1 | 74 +++++++++++++++++++++++++++++++++--- eng/publish-assets.ps1 | 30 ++++++--------- global.json | 2 +- 5 files changed, 119 insertions(+), 29 deletions(-) diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml index 5b3c23bacb903..8d6cfff732570 100644 --- a/azure-pipelines-official.yml +++ b/azure-pipelines-official.yml @@ -6,6 +6,8 @@ variables: SignType: real SkipTests: false SkipApplyOptimizationData: false + IbcSourceBranchName: 'master-vs-deps' + IbcDropId: '75e3797e1105a4da4c10dddda76c3b9398f7725a/223453/935479/1' jobs: - job: OfficialBuild @@ -19,8 +21,6 @@ jobs: timeoutInMinutes: 360 variables: - VisualStudio.MajorVersion: 16 - VisualStudio.ChannelName: 'int.d16.0stg' VisualStudio.DropName: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) steps: @@ -59,8 +59,10 @@ jobs: -officialBuildId $(Build.BuildNumber) -officialSkipTests $(SkipTests) -officialSkipApplyOptimizationData $(SkipApplyOptimizationData) + -officialSourceBranchName $(Build.SourceBranchName) + -officialIbcSourceBranchName $(IbcSourceBranchName) + -officialIbcDropId $(IbcDropId) /p:RepositoryName=$(Build.Repository.Name) - /p:VisualStudioIbcSourceBranchName="master-vs-deps" /p:VisualStudioDropAccessToken=$(System.AccessToken) /p:VisualStudioDropName=$(VisualStudio.DropName) /p:DotNetSignType=$(SignType) @@ -70,8 +72,6 @@ jobs: /p:PublishToSymbolServer=true /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:IbcOptimizationDataDir=$(Build.SourcesDirectory)\.o\\ - /p:VisualStudioIbcDropId=75e3797e1105a4da4c10dddda76c3b9398f7725a/223453/935479/1 displayName: Build condition: succeeded() diff --git a/eng/build-utils.ps1 b/eng/build-utils.ps1 index f884391a87763..0dcf9bdcb5d1b 100644 --- a/eng/build-utils.ps1 +++ b/eng/build-utils.ps1 @@ -8,6 +8,7 @@ $ErrorActionPreference="Stop" $VSSetupDir = Join-Path $ArtifactsDir "VSSetup\$configuration" $PackagesDir = Join-Path $ArtifactsDir "packages\$configuration" +$PublishDataUrl = "https://raw.githubusercontent.com/dotnet/roslyn/master/eng/config/PublishData.json" $binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $false } $nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { $false } @@ -22,6 +23,37 @@ function GetProjectOutputBinary([string]$fileName, [string]$projectName = "", [s return Join-Path $ArtifactsDir "bin\$projectName\$configuration\$tfm\$ridDir$publishDir$fileName" } +function GetPublishData() { + if (Test-Path variable:global:_PublishData) { + return $global:_PublishData + } + + Write-Host "Downloading $PublishDataUrl" + $content = (Invoke-WebRequest -Uri $PublishDataUrl -UseBasicParsing).Content + + return $global:_PublishData = ConvertFrom-Json $content +} + +function GetBranchPublishData([string]$branchName) { + $data = GetPublishData + + if (Get-Member -InputObject $data.branches -Name $branchName) { + return $data.branches.$branchName + } else { + return $null + } +} + +function GetReleasePublishData([string]$releaseName) { + $data = GetPublishData + + if (Get-Member -InputObject $data.releases -Name $releaseName) { + return $data.releases.$releaseName + } else { + return $null + } +} + # Handy function for executing a command in powershell and throwing if it # fails. # diff --git a/eng/build.ps1 b/eng/build.ps1 index 0b521c165b8aa..9395cea571982 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -46,6 +46,9 @@ param ( [string]$officialBuildId = "", [string]$officialSkipApplyOptimizationData = "", [string]$officialSkipTests = "", + [string]$officialSourceBranchName = "", + [string]$officialIbcSourceBranchName = "", + [string]$officialIbcDropId = "", # Test actions [switch]$test32, @@ -100,6 +103,11 @@ function Print-Usage() { Write-Host " -officialBuildId An official build id, e.g. 20190102.3" Write-Host " -officialSkipTests Pass 'true' to not run tests" Write-Host " -officialSkipApplyOptimizationData Pass 'true' to not apply optimization data" + Write-Host " -officialSourceBranchName The source branch name" + Write-Host " -officialIbcDropId IBC data drop to use (e.g. '20190210.1/935479/1')." + Write-Host " 'default' for the most recent available for the branch." + Write-Host " -officialIbcSourceBranchName IBC source branch (e.g. 'master-vs-deps')" + Write-Host " 'default' to select branch based on eng/config/PublishData.json." Write-Host "" Write-Host "Command line arguments starting with '/p:' are passed through to MSBuild." } @@ -133,6 +141,9 @@ function Process-Arguments() { OfficialBuildOnly "officialSkipTests" OfficialBuildOnly "officialSkipApplyOptimizationData" + OfficialBuildOnly "officialSourceBranchName" + OfficialBuildOnly "officialIbcDropId" + OfficialBuildOnly "officialIbcSourceBranchName" if ($officialBuildId) { $script:useGlobalNuGetCache = $false @@ -192,10 +203,14 @@ function BuildSolution() { $toolsetBuildProj = InitializeToolset $quietRestore = !$ci $testTargetFrameworks = if ($testCoreClr) { "netcoreapp2.1" } else { "" } + $ibcSourceBranchName = GetIbcSourceBranchName + $ibcDropId = if ($officialIbcDropId -ne "default") { $officialIbcDropId } else { "" } - # Do not set these properties to true explicitly, since that would override values set in projects. + # Do not set this property to true explicitly, since that would override values set in projects. $suppressExtensionDeployment = if (!$deployExtensions) { "/p:DeployExtension=false" } else { "" } - $suppressPartialNgenOptimization = if (!$applyOptimizationData) { "/p:ApplyPartialNgenOptimization=false" } else { "" } + + # Workaround for some machines in the AzDO pool not allowing long paths (%5c is msbuild escaped backslash) + $ibcDir = Join-Path $RepoRoot ".o%5c" # Setting /p:TreatWarningsAsErrors=true is a workaround for https://github.com/Microsoft/msbuild/issues/3062. # We don't pass /warnaserror to msbuild ($warnAsError is set to $false by default above), but set @@ -221,13 +236,62 @@ function BuildSolution() { /p:QuietRestoreBinaryLog=$binaryLog ` /p:TestTargetFrameworks=$testTargetFrameworks ` /p:TreatWarningsAsErrors=true ` - $suppressPartialNgenOptimization ` + /p:VisualStudioIbcSourceBranchName=$ibcSourceBranchName ` + /p:VisualStudioIbcDropId=$ibcDropId ` + /p:EnablePartialNgenOptimization=$applyOptimizationData ` + /p:IbcOptimizationDataDir=$ibcDir ` $suppressExtensionDeployment ` @properties } + +# Get the branch that produced the IBC data this build is going to consume. +# IBC data are only merged in official built, but we want to test some of the logic in CI builds as well. +function GetIbcSourceBranchName() { + if (Test-Path variable:global:_IbcSourceBranchName) { + return $global:_IbcSourceBranchName + } + + function calculate { + $fallback = "master-vs-deps" + + if (!$officialIbcSourceBranchName) { + return $fallback + } + + if ($officialIbcSourceBranchName -ne "default") { + return $officialIbcSourceBranchName + } + + $branchData = GetBranchPublishData $officialSourceBranchName + if ($branchData -eq $null) { + Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using IBC data from '$fallback'. " + ` + "Override by setting IbcSourceBranchName build variable." -ForegroundColor Yellow + return $fallback + } + + if (Test-Member $branchData "ibcSourceBranch") { + return $branchData.ibcSourceBranch + } + + return $officialSourceBranchName + } + + return $global:_IbcSourceBranchName = calculate +} + # Set VSO variables used by MicroBuildBuildVSBootstrapper pipeline task -function Set-OptProfVariables() { +function SetVisualStudioBootstrapperBuildArgs() { + $branchData = GetBranchPublishData (GetIbcSourceBranchName) + + # VS branch name is e.g. "lab/d16.0stg", "rel/d15.9", "lab/ml", etc. + $vsBranchSimpleName = $branchData.vsBranch.Split('/')[-1] + $vsMajorVersion = $branchData.vsMajorVersion + $vsChannel = "int.$vsBranchSimpleName" + + Write-Host "##vso[task.setvariable variable=VisualStudio.MajorVersion;]$vsMajorVersion" + Write-Host "##vso[task.setvariable variable=VisualStudio.ChannelName;]$vsChannel" + $insertionDir = Join-Path $VSSetupDir "Insertion" $manifestList = [string]::Join(',', (Get-ChildItem "$insertionDir\*.vsman")) Write-Host "##vso[task.setvariable variable=VisualStudio.SetupManifestList;]$manifestList" @@ -444,7 +508,7 @@ try { } if ($ci -and $build -and $msbuildEngine -eq "vs") { - Set-OptProfVariables + SetVisualStudioBootstrapperBuildArgs } if ($testDesktop -or $testVsi -or $testIOperation) { diff --git a/eng/publish-assets.ps1 b/eng/publish-assets.ps1 index 04df43cfbabea..0344f0cfda5b7 100644 --- a/eng/publish-assets.ps1 +++ b/eng/publish-assets.ps1 @@ -126,11 +126,6 @@ function Publish-Entry($publishData, [switch]$isBranch) { exit 0 } -function Test-Member($obj, [string]$name) { - $value = Get-Member -Name $name -InputObject $obj - return $value -ne $null -} - try { if ($configuration -eq "") { Write-Host "Must provide the build configuration with -configuration" @@ -141,29 +136,28 @@ try { $dotnet = Ensure-DotnetSdk - Write-Host "Downloading PublishData.json" - $publishFileContent = (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/dotnet/roslyn/master/eng/config/PublishData.json" -UseBasicParsing).Content - $data = ConvertFrom-Json $publishFileContent - if ($branchName -ne "" -and $releaseName -ne "") { Write-Host "Can only specify -branchName or -releaseName, not both" exit 1 } - elseif ($branchName -ne "") { - if (-not (Test-Member $data.branches $branchName)) { - Write-Host "$branchName is not listed for publishing" + + if ($branchName -ne "") { + $data = GetBranchPublishData $branchName + if ($data -eq $null) { + Write-Host "Branch $branchName not listed for publishing." exit 0 } - Publish-Entry $data.branches.$branchName -isBranch:$true + Publish-Entry $data -isBranch:$true } - elseif ($releaseName -ne "") { - if (-not (Test-Member $data.releases $releaseName)) { - Write-Host "$releaseName is not a valid release" - exit 1 + elseif ($releaseName -ne "") { + $data = GetReleasePublishData $releaseName + if ($data -eq $null) { + Write-Host "Release $releaseName not listed for publishing." + exit 1 } - Publish-Entry $data.releases.$releaseName -isBranch:$false + Publish-Entry $data -isBranch:$false } else { Write-Host "Need to specify -branchName or -releaseName" diff --git a/global.json b/global.json index 2234f323e3e6b..e07b9f2bf80f8 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19104.18" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19113.4" } }