Move PR validation logic from the official build to a separate yml

JoeRobich · JoeRobich · commit e45bcd0e30b1 · 2021-06-11T17:20:51.000-07:00
diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml
@@ -9,7 +9,6 @@ resources:
 #  SkipTests: false
 #  SkipApplyOptimizationData: false
 #  IbcDrop: 'default'
-#  PRNumber: 'default'
 
 # The variables `_DotNetArtifactsCategory` and `_DotNetValidationArtifactsCategory` are required for proper publishing of build artifacts. See https://github.com/dotnet/roslyn/pull/38259
 variables:
@@ -19,14 +18,8 @@ variables:
     value: .NETCoreValidation
   - group: DotNet-Roslyn-SDLValidation-Params
 
-  # To retrieve OptProf data we need to authenticate to the VS drop storage.
-  # If the pipeline is running in DevDiv, the account has access to the VS drop storage.
-  # Get $AccessToken-dotnet-build-bot-public-repo from DotNet-GitHub-Versions-Repo-Write
-  - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-    - group: DotNet-GitHub-Versions-Repo-Write
-    - name: _DevDivDropAccessToken
-      value: $(System.AccessToken)
   # If the pipeline is running in dnceng:
+  # To retrieve OptProf data we need to authenticate to the VS drop storage.
   # Get access token with $dn-bot-devdiv-drop-rw-code-rw and dn-bot-dnceng-build-rw-code-rw from DotNet-VSTS-Infra-Access
   # Get $dotnetfeed-storage-access-key-1 from DotNet-Blob-Feed
   # Get $microsoft-symbol-server-pat and $symweb-symbol-server-pat from DotNet-Symbol-Server-Pats
@@ -54,31 +47,13 @@ stages:
   - job: OfficialBuild
     displayName: Official Build
     timeoutInMinutes: 360
-    # Conditionally set build pool so we can share this YAML when building with different pipeline (devdiv vs dnceng)
+    # Conditionally set build pool so we can share this YAML when building with different pipeline
     pool:
-      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-        name: VSEngSS-MicroBuild2017
-        demands:
-        - msbuild
-        - visualstudio
-        - DotNetFramework
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: NetCoreInternal-Pool
         queue: BuildPool.Server.Amd64.VS2017
 
     steps:
-    # Make sure our two pipelines generate builds with distinct build numbers to avoid confliction.
-    # i.e. DevDiv builds will have even rev# whereas dnceng builds will be odd.
-    - task: PowerShell@2
-      displayName: Update BuildNumber
-      inputs:
-        filePath: 'eng\update-build-number.ps1'
-        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-          arguments: '-buildNumber $(Build.BuildNumber) -oddOrEven even'
-        ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-          arguments: '-buildNumber $(Build.BuildNumber) -oddOrEven odd'
-      condition: eq(variables['System.JobAttempt'], '1')
-
     - powershell: Write-Host "##vso[task.setvariable variable=SourceBranchName]$('$(Build.SourceBranch)'.Substring('refs/heads/'.Length))"
       displayName: Setting SourceBranchName variable
       condition: succeeded()
@@ -88,23 +63,7 @@ stages:
       inputs:
         type: 'Build'
         tags: 'OfficialBuild'
-      condition: and(succeeded(), endsWith(variables['SourceBranchName'], '-vs-deps'), eq(variables['PRNumber'], 'default'))
-
-    - task: tagBuildOrRelease@0
-      displayName: Tag PR validation build
-      inputs:
-        type: 'Build'
-        tags: |
-          PRValidationBuild
-          PRNumber:$(PRNumber)
-      condition: and(succeeded(), ne(variables['PRNumber'], 'default'))
-
-    - task: PowerShell@2
-      displayName: Setup branch for insertion validation
-      inputs:
-        filePath: 'eng\setup-pr-validation.ps1'
-        arguments: '-sourceBranchName $(SourceBranchName) -prNumber $(PRNumber)'
-      condition: and(succeeded(), ne(variables['PRNumber'], 'default'))
+      condition: and(succeeded(), endsWith(variables['SourceBranchName'], '-vs-deps'))
 
     - task: tagBuildOrRelease@0
       displayName: Tag main validation build
@@ -132,20 +91,16 @@ stages:
     # Authenticate with service connections to be able to publish packages to external nuget feeds.
     - task: NuGetAuthenticate@0
       inputs:
-        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-          nuGetServiceConnections: azure-public/vs-impl, azure-public/vssdk
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:
           nuGetServiceConnections: azure-public/vs-impl, azure-public/vssdk, devdiv/engineering
 
-    # Needed because the dnceng image we build on fails the next task without it
+    # Needed because the build fails the NuGet Tools restore without it
     - task: UseDotNet@2
       displayName: 'Use .NET Core sdk'
       inputs:
         packageType: sdk
         useGlobalJson: true
         workingDirectory: '$(Build.SourcesDirectory)'
-        installationPath: '$(Build.SourcesDirectory)/.dotnet'
-      condition: eq(variables['System.TeamProject'], 'internal')
 
     # Needed to restore the Microsoft.DevDiv.Optimization.Data.PowerShell package
     - task: NuGetCommand@2
@@ -202,7 +157,7 @@ stages:
       inputs:
         filePath: 'eng\publish-assets.ps1'
         arguments: '-configuration $(BuildConfiguration) -branchName "$(SourceBranchName)"'
-      condition: and(succeeded(), eq(variables['PRNumber'], 'default'))
+      condition: succeeded()
 
     # Publish OptProf configuration files
     # The env variable is required to enable cross account access using PAT (dnceng -> devdiv)
@@ -272,9 +227,6 @@ stages:
         command: push
         packagesToPush: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\DevDivPackages\**\*.nupkg'
         allowPackageConflicts: true
-        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-          feedsToUse: config
-          publishVstsFeed: '97a41293-2972-4f48-8c0e-05493ae82010'
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:
           nuGetFeedType: external
           publishFeedCredentials: 'DevDiv - VS package feed'
@@ -315,26 +267,10 @@ stages:
       publishUsingPipelines: true
       dependsOn:
         - OfficialBuild
-      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-        queue:
-          name: Hosted VS2017
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         pool:
           vmImage: vs2017-win2016
 
-# We need to skip post-build stages for PR validation build, but it can only be identified by
-# the runtime variable 'PRNumber', thus this dummy stage. Also the dummy job is required
-# otherwise AzDO would just repeat jobs from previous stage.
-- stage: SetValidateDependency
-  displayName: Setup the dependency for post-build stages
-  condition: and(succeeded(), eq(variables['PRNumber'], 'default'))
-  jobs:
-  - job: Log
-    displayName: Log
-    steps:
-    - powershell: Write-Host "Setup the dependency for post-build stages."
-      displayName: Log
-
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng\common\templates\post-build\post-build.yml
     parameters:
diff --git a/azure-pipelines-validation.yml b/azure-pipelines-validation.yml
@@ -0,0 +1,209 @@
+resources:
+- repo: self
+  clean: true
+
+# Variables defined in yml cannot be overridden at queue time instead overrideable variables must be defined in the web gui.
+# Commenting out until AzDO supports something like the runtime parameters outlined here: https://github.com/Microsoft/azure-pipelines-yaml/pull/129
+#variables:
+#  SignType: test
+#  SkipTests: true
+#  SkipApplyOptimizationData: false
+#  IbcDrop: 'default'
+#  PRNumber: 'REQUIRED'
+#  CommitSHA: 'REQUIRED'
+
+# The variables `_DotNetArtifactsCategory` and `_DotNetValidationArtifactsCategory` are required for proper publishing of build artifacts. See https://github.com/dotnet/roslyn/pull/38259
+variables:
+  - name: _DotNetArtifactsCategory
+    value: .NETCore
+  - name: _DotNetValidationArtifactsCategory
+    value: .NETCoreValidation
+  - group: DotNet-Roslyn-SDLValidation-Params
+
+  # To retrieve OptProf data we need to authenticate to the VS drop storage.
+  # If the pipeline is running in DevDiv, the account has access to the VS drop storage.
+  # Get $AccessToken-dotnet-build-bot-public-repo from DotNet-GitHub-Versions-Repo-Write
+  - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+    - group: DotNet-GitHub-Versions-Repo-Write
+    - name: _DevDivDropAccessToken
+      value: $(System.AccessToken)
+
+stages:
+- stage: build
+  displayName: Build and Test
+
+  jobs:
+
+  - job: OfficialTestBuild
+    displayName: Official Test Build
+    timeoutInMinutes: 360
+    # Conditionally set build pool so we can share this YAML when building with different pipeline
+    pool:
+      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+        name: VSEngSS-MicroBuild2017
+        demands:
+        - msbuild
+        - visualstudio
+        - DotNetFramework
+
+    steps:
+    - powershell: Write-Host "##vso[task.setvariable variable=SourceBranchName]$('$(Build.SourceBranch)'.Substring('refs/heads/'.Length))"
+      displayName: Setting SourceBranchName variable
+      condition: succeeded()
+
+    - task: tagBuildOrRelease@0
+      displayName: Tag PR validation build
+      inputs:
+        type: 'Build'
+        tags: |
+          PRValidationBuild
+          PRNumber:$(PRNumber)
+      condition: succeeded()
+
+    - task: PowerShell@2
+      displayName: Setup branch for insertion validation
+      inputs:
+        filePath: 'eng\setup-pr-validation.ps1'
+        arguments: '-sourceBranchName $(SourceBranchName) -prNumber $(PRNumber) -commitSHA $(CommitSHA)'
+      condition: succeeded()
+
+    - powershell: Write-Host "##vso[task.setvariable variable=VisualStudio.DropName]Products/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranchName)/$(Build.BuildNumber)"
+      displayName: Setting VisualStudio.DropName variable
+
+    - task: NuGetToolInstaller@0
+      inputs:
+        versionSpec: '4.9.2'
+
+    # Authenticate with service connections to be able to publish packages to external nuget feeds.
+    - task: NuGetAuthenticate@0
+      inputs:
+        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+          nuGetServiceConnections: azure-public/vs-impl, azure-public/vssdk
+
+    # Needed to restore the Microsoft.DevDiv.Optimization.Data.PowerShell package
+    - task: NuGetCommand@2
+      displayName: Restore internal tools
+      inputs:
+        command: restore
+        feedsToUse: config
+        restoreSolution: 'eng\common\internal\Tools.csproj'
+        nugetConfigPath: 'NuGet.config'
+        restoreDirectory: '$(Build.SourcesDirectory)\.packages'
+
+    - task: MicroBuildSigningPlugin@2
+      inputs:
+        signType: $(SignType)
+        zipSources: false
+        # If running in dnceng, we need to use a feed different from default
+        ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+      condition: and(succeeded(), in(variables['SignType'], 'test', 'real'))
+
+    - task: PowerShell@2
+      displayName: Build
+      inputs:
+        filePath: eng/build.ps1
+        arguments: -ci
+                   -restore
+                   -build
+                   -pack
+                   -sign
+                   -publish
+                   -binaryLog
+                   -configuration $(BuildConfiguration)
+                   -officialBuildId $(Build.BuildNumber)
+                   -officialSkipTests $(SkipTests)
+                   -officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
+                   -officialSourceBranchName $(SourceBranchName)
+                   -officialIbcDrop $(IbcDrop)
+                   -officialVisualStudioDropAccessToken $(_DevDivDropAccessToken)
+                   /p:RepositoryName=$(Build.Repository.Name)
+                   /p:VisualStudioDropName=$(VisualStudio.DropName)
+                   /p:DotNetSignType=$(SignType)
+                   /p:DotNetPublishToBlobFeed=false
+                   /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+                   /p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+                   /p:PublishToSymbolServer=false
+                   /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+                   /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+                   /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
+                   /p:DotnetPublishUsingPipelines=false
+      condition: succeeded()
+
+    # Publish OptProf generated JSON files as a build artifact. This allows for easy inspection from
+    # a build execution.
+    - task: PublishBuildArtifacts@1
+      displayName: Publish OptProf Data Files
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\artifacts\OptProf\$(BuildConfiguration)\Data'
+        ArtifactName: 'OptProf Data Files'
+      condition: succeeded()
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Logs
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\artifacts\log\$(BuildConfiguration)'
+        ArtifactName: 'Build Diagnostic Files'
+        publishLocation: Container
+      continueOnError: true
+      condition: succeededOrFailed()
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Ngen Logs
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\artifacts\log\$(BuildConfiguration)\ngen'
+        ArtifactName: 'NGen Logs'
+        publishLocation: Container
+      continueOnError: true
+      condition: succeeded()
+
+    - task: PublishTestResults@2
+      displayName: Publish xUnit Test Results
+      inputs:
+        testRunner: XUnit
+        testResultsFiles: '$(Build.SourcesDirectory)\artifacts\TestResults\$(BuildConfiguration)\*.xml'
+        mergeTestResults: true
+        testRunTitle: 'Unit Tests'
+      condition: and(succeededOrFailed(), ne(variables['SkipTests'], 'true'))
+
+    # Publishes setup VSIXes to a drop.
+    # Note: The insertion tool looks for the display name of this task in the logs.
+    - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1
+      displayName: Upload VSTS Drop
+      inputs:
+        DropName: $(VisualStudio.DropName)
+        DropFolder: 'artifacts\VSSetup\$(BuildConfiguration)\Insertion'
+        AccessToken: $(_DevDivDropAccessToken)
+      condition: succeeded()
+
+    # Publish insertion packages to CoreXT store.
+    - task: NuGetCommand@2
+      displayName: Publish CoreXT Packages
+      inputs:
+        command: push
+        packagesToPush: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\DevDivPackages\**\*.nupkg'
+        allowPackageConflicts: true
+        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+          feedsToUse: config
+          publishVstsFeed: '97a41293-2972-4f48-8c0e-05493ae82010'
+      condition: succeeded()
+
+    # Publish an artifact that the RoslynInsertionTool is able to find by its name.
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Artifact VSSetup
+      inputs:
+        PathtoPublish: 'artifacts\VSSetup\$(BuildConfiguration)'
+        ArtifactName: 'VSSetup'
+      condition: succeeded()
+
+    # Publish Asset Manifests for Build Asset Registry job
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Asset Manifests
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/AssetManifest'
+        ArtifactName: AssetManifests
+      condition: succeeded()
+
+    - task: ms-vseng.MicroBuildTasks.521a94ea-9e68-468a-8167-6dcf361ea776.MicroBuildCleanup@1
+      displayName: Perform Cleanup Tasks
+      condition: succeededOrFailed()
diff --git a/eng/setup-pr-validation.ps1 b/eng/setup-pr-validation.ps1
