Join assets from verticals in the final join point

eng/pipelines/templates/jobs/vmr-build.yml

@@ -172,6 +172,13 @@ jobs:
       displayName: Publish Artifacts
       sbomEnabled: true
 
+    # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
+    - output: buildArtifacts
+      PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
+      ArtifactName: VerticalManifests
+      displayName: Publish Vertical Manifest
+      sbomEnabled: false
+
     - ${{ if not(parameters.isBuiltFromVmr) }}:
       - output: pipelineArtifact
         displayName: Upload failed patches
@@ -272,7 +279,7 @@ jobs:
     - script: |
         set extraBuildProperties=
         if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }}
-        call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} %extraBuildProperties% ${{ parameters.extraProperties }}
+        call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }}
       displayName: Build
       workingDirectory: ${{ variables.sourcesPath }}
       env:
@@ -283,7 +290,7 @@ jobs:
 
     - ${{ if eq(parameters.runTests, 'True') }}:
       - script: |
-          call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
+          call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }}
         displayName: Run Tests
         workingDirectory: ${{ variables.sourcesPath }}
         timeoutInMinutes: ${{ variables.runTestsTimeout }}
@@ -381,6 +388,8 @@ jobs:
           extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
         fi
 
+        extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
+
         buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
 
         # Only use Docker when a container is specified
@@ -456,6 +465,8 @@ jobs:
             customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
           fi
 
+          extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
+
           if [[ -n "${{ parameters.extraProperties }}" ]]; then
             extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
           fi
@@ -597,6 +608,19 @@ jobs:
       TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
     displayName: Copy artifacts to Artifact Staging Directory
 
+  - ${{ if eq(parameters.targetOS, 'windows') }}:
+    - powershell: |
+        $sourcePath = "$(sourcesPath)/artifacts/VerticalManifest.xml"
+        $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
+        New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null
+        Copy-Item $sourcePath -Destination $targetPath -Force
+      displayName: Copy vertical manifest to Artifact Staging Directory
+  - ${{ else }}:
+    - script: |
+        mkdir -p "$(Build.ArtifactStagingDirectory)/manifests"
+        cp "$(sourcesPath)/artifacts/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
+      displayName: Copy vertical manifest to Artifact Staging Directory
+
   # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
   - ${{ if ne(parameters.buildSourceOnly, 'true') }}:
     - task: CopyFiles@2
@@ -610,3 +634,11 @@ jobs:
       artifact: $(Agent.JobName)_Artifacts$(artifactsSuffix)
       displayName: Publish Artifacts
       continueOnError: true
+
+    # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
+        ArtifactName: VerticalManifests
+      displayName: Publish Vertical Manifest
+      condition: succeededOrFailed()

eng/pipelines/templates/stages/vmr-final-join.yml

@@ -0,0 +1,52 @@
+parameters:
+# Branch of the VMR to use (to push to for internal builds)
+- name: vmrBranch
+  type: string
+  default: $(Build.SourceBranch)
+
+- name: pool_Windows
+  type: object
+  default:
+    name: $(defaultPoolName)
+    image: $(poolImage_Windows)
+    demands: ImageOverride -equals $(poolImage_Windows)
+    os: windows
+
+stages:
+- stage: VMR_Final_Join
+  displayName: VMR Final Join
+  dependsOn: VMR_Vertical_Build
+  condition: succeededOrFailed()
+  variables:
+  - template: ../variables/vmr-build.yml
+    parameters:
+      vmrBranch: ${{ parameters.vmrBranch }}
+
+  jobs:
+  - job: FinalJoin
+    displayName: Final Build Pass
+    pool: ${{ parameters.pool_Windows }}
+    timeoutInMinutes: 240
+    templateContext:
+      outputs:
+      - output: buildArtifacts
+        PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/MergedManifest.xml
+        ArtifactName: AssetManifests
+        displayName: Publish Merged Manifest
+        sbomEnabled: false
+      - output: buildArtifacts
+        PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/assets
+        ArtifactName: BlobArtifacts
+        displayName: Publish Blob Artifacts
+        sbomEnabled: false
+      - output: buildArtifacts
+        PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages
+        ArtifactName: PackageArtifacts
+        displayName: Publish Package Artifacts
+        sbomEnabled: false
+    steps:
+    - template: ../steps/vmr-join-verticals.yml
+      parameters:
+        dotNetBuildPass: final
+        primaryDependentJob: Windows_x64
+        outputFolder: $(Build.ArtifactStagingDirectory)/artifacts

eng/pipelines/templates/steps/vmr-join-verticals.yml

@@ -0,0 +1,37 @@
+parameters:
+- name: dotNetBuildPass
+  type: string
+  default: final
+
+- name: primaryDependentJob
+  type: string
+  default: Windows_x64
+
+- name: outputFolder
+  type: string
+  default: $(Build.ArtifactStagingDirectory)/artifacts
+
+steps:
+- task: DownloadBuildArtifacts@1
+  inputs:
+    artifactName: 'VerticalManifests'
+    downloadPath: $(Build.ArtifactStagingDirectory)
+    checkDownloadedFiles: true
+
+- task: DownloadPipelineArtifact@2
+  inputs:
+    artifactName: ${{ parameters.primaryDependentJob }}_Artifacts
+    targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
+    checkDownloadedFiles: true
+
+- powershell: eng/join-verticals.ps1
+    /p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests 
+    /p:MainVertical=${{ parameters.primaryDependentJob }}
+    /p:DotNetBuildPass=${{ parameters.dotNetBuildPass }}
+    /p:BuildId=$(Build.BuildId)
+    /p:AzureDevOpsToken=$(System.AccessToken) 
+    /p:AzureDevOpsBaseUri=$(System.CollectionUri)
+    /p:AzureDevOpsProject=$(System.TeamProject)
+    /p:MainVerticalArtifactsFolder=$(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
+    /p:OutputFolder=${{ parameters.outputFolder }}
+  displayName: Join Verticals

src/SourceBuild/content/build.proj

@@ -30,9 +30,12 @@
     <PropertyGroup>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' == 'true'">source-build</BuildModeInfoText>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' != 'true'">non-source-build</BuildModeInfoText>
+      <BuildPassInfoText Condition="'$(DotNetBuildPass)' == ''">1</BuildPassInfoText>
+      <BuildPassInfoText Condition="'$(DotNetBuildPass)' != ''">$(DotNetBuildPass)</BuildPassInfoText>
     </PropertyGroup>
 
     <Message Text="Build Mode: $(BuildModeInfoText)" Importance="high" />
+    <Message Text="Build Pass: $(BuildPassInfoText)" Importance="high" />
     <Message Text="Build Environment: $(TargetArchitecture) $(Configuration) $(TargetOS) $(TargetRid)" Importance="high" />
     <Message Condition="'$(EnablePoison)' == 'true'" Text="Crossgen is disabled due to incompatibility with poison checks." Importance="high" />
   </Target>

src/SourceBuild/content/eng/join-verticals.proj

@@ -0,0 +1,39 @@
+<Project Sdk="Microsoft.Build.Traversal">
+
+    <PropertyGroup>
+      <TargetFramework>$(NetCurrent)</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" BuildInParallel="true" />
+    </ItemGroup>
+
+    <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+    <Target Name="JoinVerticals" DependsOnTargets="ResolveProjectReferences" AfterTargets="Build">
+      <Error Condition="'$(MainVertical)' == ''" Text="MainVertical is not set." />
+      <Error Condition="'$(VerticalManifestsPath)' == ''" Text="VerticalManifestsPath is not set." />
+      <Error Condition="'$(BuildId)' == ''" Text="BuildId is not set." />
+      <Error Condition="'$(AzureDevOpsBaseUri)' == ''" Text="AzureDevOpsBaseUri is not set." />
+      <Error Condition="'$(AzureDevOpsProject)' == ''" Text="AzureDevOpsProject is not set." />
+      <Error Condition="'$(MainVerticalArtifactsFolder)' == ''" Text="MainVerticalArtifactsFolder is not set." />
+      <Error Condition="'$(OutputFolder)' == ''" Text="OutputFolder is not set." />
+
+      <ItemGroup>
+        <VerticalManifest 
+          Include="$(VerticalManifestsPath)\*.xml"
+          Exclude="$(VerticalManifestsPath)\*Mono*.xml;$(VerticalManifestsPath)\*Pgo*.xml;$(VerticalManifestsPath)\*Shortstack*.xml;$(VerticalManifestsPath)\*DevVersions*.xml" />
+      </ItemGroup>
+
+      <!-- AzureDevOpsToken shouldn't be set when running in dnceng-public -->
+      <Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals 
+        VerticalManifest="@(VerticalManifest)"
+        MainVertical="$(MainVertical)"
+        BuildId="$(BuildId)"
+        AzureDevOpsToken="$(AzureDevOpsToken)"
+        AzureDevOpsBaseUri="$(AzureDevOpsBaseUri)"
+        AzureDevOpsProject="$(AzureDevOpsProject)"
+        MainVerticalArtifactsFolder="$(MainVerticalArtifactsFolder)"
+        OutputFolder="$(OutputFolder)" />
+    </Target>
+
+  </Project>

src/SourceBuild/content/eng/join-verticals.ps1

@@ -0,0 +1,33 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+  [string][Alias('v')]$verbosity = "minimal",
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+)
+
+$useGlobalNuGetCache=$false
+$ci = $true
+
+. $PSScriptRoot\common\tools.ps1
+
+$project = Join-Path $EngRoot "join-verticals.proj"
+$arguments = @()
+$targets = "/t:JoinVerticals"
+
+try {
+  $bl = '/bl:' + (Join-Path $LogDir 'JoinVerticals.binlog')
+
+  MSBuild -restore `
+    $project `
+    $bl `
+    $targets `
+    /p:Configuration=Release `
+    @properties `
+    @arguments
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Build' -Message $_
+  ExitWithExitCode 1
+}
+
+ExitWithExitCode 0

src/SourceBuild/content/eng/merge-asset-manifests.proj

@@ -21,7 +21,8 @@
     <Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests 
       AssetManifest="@(RepoAssetManifest)" 
       MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)"
-      VmrBuildNumber="$(BUILD_BUILDNUMBER)" />
+      VmrBuildNumber="$(BUILD_BUILDNUMBER)"
+      VerticalName="$(VerticalName)" />
   </Target>
 
 </Project>

src/SourceBuild/content/eng/pipelines/ci.yml

@@ -95,3 +95,6 @@ extends:
           scope: lite
         ${{ else }}:
           scope: full
+
+    - ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
+      - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self

src/SourceBuild/content/eng/pipelines/pr.yml

@@ -48,3 +48,6 @@ stages:
       scope: lite
     ${{ else }}:
       scope: full
+
+- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
+  - template: /src/sdk/eng/pipelines/templates/stages/vmr-post-build.yml@self