Skip to content

Commit

Permalink
Move PR validation logic from the official build to a separate yml
Browse files Browse the repository at this point in the history
  • Loading branch information
JoeRobich committed Jun 12, 2021
1 parent b240d31 commit e45bcd0
Show file tree
Hide file tree
Showing 3 changed files with 237 additions and 76 deletions.
74 changes: 5 additions & 69 deletions azure-pipelines-official.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ resources:
# SkipTests: false
# SkipApplyOptimizationData: false
# IbcDrop: 'default'
# PRNumber: 'default'

# The variables `_DotNetArtifactsCategory` and `_DotNetValidationArtifactsCategory` are required for proper publishing of build artifacts. See https://github.com/dotnet/roslyn/pull/38259
variables:
Expand All @@ -19,14 +18,8 @@ variables:
value: .NETCoreValidation
- group: DotNet-Roslyn-SDLValidation-Params

# To retrieve OptProf data we need to authenticate to the VS drop storage.
# If the pipeline is running in DevDiv, the account has access to the VS drop storage.
# Get $AccessToken-dotnet-build-bot-public-repo from DotNet-GitHub-Versions-Repo-Write
- ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
- group: DotNet-GitHub-Versions-Repo-Write
- name: _DevDivDropAccessToken
value: $(System.AccessToken)
# If the pipeline is running in dnceng:
# To retrieve OptProf data we need to authenticate to the VS drop storage.
# Get access token with $dn-bot-devdiv-drop-rw-code-rw and dn-bot-dnceng-build-rw-code-rw from DotNet-VSTS-Infra-Access
# Get $dotnetfeed-storage-access-key-1 from DotNet-Blob-Feed
# Get $microsoft-symbol-server-pat and $symweb-symbol-server-pat from DotNet-Symbol-Server-Pats
Expand Down Expand Up @@ -54,31 +47,13 @@ stages:
- job: OfficialBuild
displayName: Official Build
timeoutInMinutes: 360
# Conditionally set build pool so we can share this YAML when building with different pipeline (devdiv vs dnceng)
# Conditionally set build pool so we can share this YAML when building with different pipeline
pool:
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: VSEngSS-MicroBuild2017
demands:
- msbuild
- visualstudio
- DotNetFramework
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCoreInternal-Pool
queue: BuildPool.Server.Amd64.VS2017

steps:
# Make sure our two pipelines generate builds with distinct build numbers to avoid confliction.
# i.e. DevDiv builds will have even rev# whereas dnceng builds will be odd.
- task: PowerShell@2
displayName: Update BuildNumber
inputs:
filePath: 'eng\update-build-number.ps1'
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
arguments: '-buildNumber $(Build.BuildNumber) -oddOrEven even'
${{ if eq(variables['System.TeamProject'], 'internal') }}:
arguments: '-buildNumber $(Build.BuildNumber) -oddOrEven odd'
condition: eq(variables['System.JobAttempt'], '1')

- powershell: Write-Host "##vso[task.setvariable variable=SourceBranchName]$('$(Build.SourceBranch)'.Substring('refs/heads/'.Length))"
displayName: Setting SourceBranchName variable
condition: succeeded()
Expand All @@ -88,23 +63,7 @@ stages:
inputs:
type: 'Build'
tags: 'OfficialBuild'
condition: and(succeeded(), endsWith(variables['SourceBranchName'], '-vs-deps'), eq(variables['PRNumber'], 'default'))

- task: tagBuildOrRelease@0
displayName: Tag PR validation build
inputs:
type: 'Build'
tags: |
PRValidationBuild
PRNumber:$(PRNumber)
condition: and(succeeded(), ne(variables['PRNumber'], 'default'))

- task: PowerShell@2
displayName: Setup branch for insertion validation
inputs:
filePath: 'eng\setup-pr-validation.ps1'
arguments: '-sourceBranchName $(SourceBranchName) -prNumber $(PRNumber)'
condition: and(succeeded(), ne(variables['PRNumber'], 'default'))
condition: and(succeeded(), endsWith(variables['SourceBranchName'], '-vs-deps'))

- task: tagBuildOrRelease@0
displayName: Tag main validation build
Expand Down Expand Up @@ -132,20 +91,16 @@ stages:
# Authenticate with service connections to be able to publish packages to external nuget feeds.
- task: NuGetAuthenticate@0
inputs:
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
nuGetServiceConnections: azure-public/vs-impl, azure-public/vssdk
${{ if eq(variables['System.TeamProject'], 'internal') }}:
nuGetServiceConnections: azure-public/vs-impl, azure-public/vssdk, devdiv/engineering

# Needed because the dnceng image we build on fails the next task without it
# Needed because the build fails the NuGet Tools restore without it
- task: UseDotNet@2
displayName: 'Use .NET Core sdk'
inputs:
packageType: sdk
useGlobalJson: true
workingDirectory: '$(Build.SourcesDirectory)'
installationPath: '$(Build.SourcesDirectory)/.dotnet'
condition: eq(variables['System.TeamProject'], 'internal')

# Needed to restore the Microsoft.DevDiv.Optimization.Data.PowerShell package
- task: NuGetCommand@2
Expand Down Expand Up @@ -202,7 +157,7 @@ stages:
inputs:
filePath: 'eng\publish-assets.ps1'
arguments: '-configuration $(BuildConfiguration) -branchName "$(SourceBranchName)"'
condition: and(succeeded(), eq(variables['PRNumber'], 'default'))
condition: succeeded()

# Publish OptProf configuration files
# The env variable is required to enable cross account access using PAT (dnceng -> devdiv)
Expand Down Expand Up @@ -272,9 +227,6 @@ stages:
command: push
packagesToPush: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\DevDivPackages\**\*.nupkg'
allowPackageConflicts: true
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
feedsToUse: config
publishVstsFeed: '97a41293-2972-4f48-8c0e-05493ae82010'
${{ if eq(variables['System.TeamProject'], 'internal') }}:
nuGetFeedType: external
publishFeedCredentials: 'DevDiv - VS package feed'
Expand Down Expand Up @@ -315,26 +267,10 @@ stages:
publishUsingPipelines: true
dependsOn:
- OfficialBuild
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
queue:
name: Hosted VS2017
${{ if eq(variables['System.TeamProject'], 'internal') }}:
pool:
vmImage: vs2017-win2016

# We need to skip post-build stages for PR validation build, but it can only be identified by
# the runtime variable 'PRNumber', thus this dummy stage. Also the dummy job is required
# otherwise AzDO would just repeat jobs from previous stage.
- stage: SetValidateDependency
displayName: Setup the dependency for post-build stages
condition: and(succeeded(), eq(variables['PRNumber'], 'default'))
jobs:
- job: Log
displayName: Log
steps:
- powershell: Write-Host "Setup the dependency for post-build stages."
displayName: Log

- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- template: eng\common\templates\post-build\post-build.yml
parameters:
Expand Down
209 changes: 209 additions & 0 deletions azure-pipelines-validation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
resources:
- repo: self
clean: true

# Variables defined in yml cannot be overridden at queue time instead overrideable variables must be defined in the web gui.
# Commenting out until AzDO supports something like the runtime parameters outlined here: https://github.com/Microsoft/azure-pipelines-yaml/pull/129
#variables:
# SignType: test
# SkipTests: true
# SkipApplyOptimizationData: false
# IbcDrop: 'default'
# PRNumber: 'REQUIRED'
# CommitSHA: 'REQUIRED'

# The variables `_DotNetArtifactsCategory` and `_DotNetValidationArtifactsCategory` are required for proper publishing of build artifacts. See https://github.com/dotnet/roslyn/pull/38259
variables:
- name: _DotNetArtifactsCategory
value: .NETCore
- name: _DotNetValidationArtifactsCategory
value: .NETCoreValidation
- group: DotNet-Roslyn-SDLValidation-Params

# To retrieve OptProf data we need to authenticate to the VS drop storage.
# If the pipeline is running in DevDiv, the account has access to the VS drop storage.
# Get $AccessToken-dotnet-build-bot-public-repo from DotNet-GitHub-Versions-Repo-Write
- ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
- group: DotNet-GitHub-Versions-Repo-Write
- name: _DevDivDropAccessToken
value: $(System.AccessToken)

stages:
- stage: build
displayName: Build and Test

jobs:

- job: OfficialTestBuild
displayName: Official Test Build
timeoutInMinutes: 360
# Conditionally set build pool so we can share this YAML when building with different pipeline
pool:
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: VSEngSS-MicroBuild2017
demands:
- msbuild
- visualstudio
- DotNetFramework

steps:
- powershell: Write-Host "##vso[task.setvariable variable=SourceBranchName]$('$(Build.SourceBranch)'.Substring('refs/heads/'.Length))"
displayName: Setting SourceBranchName variable
condition: succeeded()

- task: tagBuildOrRelease@0
displayName: Tag PR validation build
inputs:
type: 'Build'
tags: |
PRValidationBuild
PRNumber:$(PRNumber)
condition: succeeded()

- task: PowerShell@2
displayName: Setup branch for insertion validation
inputs:
filePath: 'eng\setup-pr-validation.ps1'
arguments: '-sourceBranchName $(SourceBranchName) -prNumber $(PRNumber) -commitSHA $(CommitSHA)'
condition: succeeded()

- powershell: Write-Host "##vso[task.setvariable variable=VisualStudio.DropName]Products/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranchName)/$(Build.BuildNumber)"
displayName: Setting VisualStudio.DropName variable

- task: NuGetToolInstaller@0
inputs:
versionSpec: '4.9.2'

# Authenticate with service connections to be able to publish packages to external nuget feeds.
- task: NuGetAuthenticate@0
inputs:
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
nuGetServiceConnections: azure-public/vs-impl, azure-public/vssdk

# Needed to restore the Microsoft.DevDiv.Optimization.Data.PowerShell package
- task: NuGetCommand@2
displayName: Restore internal tools
inputs:
command: restore
feedsToUse: config
restoreSolution: 'eng\common\internal\Tools.csproj'
nugetConfigPath: 'NuGet.config'
restoreDirectory: '$(Build.SourcesDirectory)\.packages'

- task: MicroBuildSigningPlugin@2
inputs:
signType: $(SignType)
zipSources: false
# If running in dnceng, we need to use a feed different from default
${{ if eq(variables['System.TeamProject'], 'internal') }}:
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
condition: and(succeeded(), in(variables['SignType'], 'test', 'real'))

- task: PowerShell@2
displayName: Build
inputs:
filePath: eng/build.ps1
arguments: -ci
-restore
-build
-pack
-sign
-publish
-binaryLog
-configuration $(BuildConfiguration)
-officialBuildId $(Build.BuildNumber)
-officialSkipTests $(SkipTests)
-officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
-officialSourceBranchName $(SourceBranchName)
-officialIbcDrop $(IbcDrop)
-officialVisualStudioDropAccessToken $(_DevDivDropAccessToken)
/p:RepositoryName=$(Build.Repository.Name)
/p:VisualStudioDropName=$(VisualStudio.DropName)
/p:DotNetSignType=$(SignType)
/p:DotNetPublishToBlobFeed=false
/p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
/p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
/p:PublishToSymbolServer=false
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
/p:DotnetPublishUsingPipelines=false
condition: succeeded()

# Publish OptProf generated JSON files as a build artifact. This allows for easy inspection from
# a build execution.
- task: PublishBuildArtifacts@1
displayName: Publish OptProf Data Files
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\OptProf\$(BuildConfiguration)\Data'
ArtifactName: 'OptProf Data Files'
condition: succeeded()

- task: PublishBuildArtifacts@1
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\log\$(BuildConfiguration)'
ArtifactName: 'Build Diagnostic Files'
publishLocation: Container
continueOnError: true
condition: succeededOrFailed()

- task: PublishBuildArtifacts@1
displayName: Publish Ngen Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\log\$(BuildConfiguration)\ngen'
ArtifactName: 'NGen Logs'
publishLocation: Container
continueOnError: true
condition: succeeded()

- task: PublishTestResults@2
displayName: Publish xUnit Test Results
inputs:
testRunner: XUnit
testResultsFiles: '$(Build.SourcesDirectory)\artifacts\TestResults\$(BuildConfiguration)\*.xml'
mergeTestResults: true
testRunTitle: 'Unit Tests'
condition: and(succeededOrFailed(), ne(variables['SkipTests'], 'true'))

# Publishes setup VSIXes to a drop.
# Note: The insertion tool looks for the display name of this task in the logs.
- task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1
displayName: Upload VSTS Drop
inputs:
DropName: $(VisualStudio.DropName)
DropFolder: 'artifacts\VSSetup\$(BuildConfiguration)\Insertion'
AccessToken: $(_DevDivDropAccessToken)
condition: succeeded()

# Publish insertion packages to CoreXT store.
- task: NuGetCommand@2
displayName: Publish CoreXT Packages
inputs:
command: push
packagesToPush: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\DevDivPackages\**\*.nupkg'
allowPackageConflicts: true
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
feedsToUse: config
publishVstsFeed: '97a41293-2972-4f48-8c0e-05493ae82010'
condition: succeeded()

# Publish an artifact that the RoslynInsertionTool is able to find by its name.
- task: PublishBuildArtifacts@1
displayName: Publish Artifact VSSetup
inputs:
PathtoPublish: 'artifacts\VSSetup\$(BuildConfiguration)'
ArtifactName: 'VSSetup'
condition: succeeded()

# Publish Asset Manifests for Build Asset Registry job
- task: PublishBuildArtifacts@1
displayName: Publish Asset Manifests
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/AssetManifest'
ArtifactName: AssetManifests
condition: succeeded()

- task: ms-vseng.MicroBuildTasks.521a94ea-9e68-468a-8167-6dcf361ea776.MicroBuildCleanup@1
displayName: Perform Cleanup Tasks
condition: succeededOrFailed()
Loading

0 comments on commit e45bcd0

Please sign in to comment.