[ci] Improve maestro artifact publishing (#20665)

pjcollins · dalexsoto · web-flow · commit 2abbaf900be8 · 2024-06-04T12:40:53.000-04:00
The steps used to publish build asset information to maestro have been updated. The post build stage has been disabled for main and .NET 9, as maestro manifest generation and channel promotion will now run as part of the "Prepare .NET Release" stage. With the new `PushToAzureDevOpsArtifacts` task the build pipeline should now create all of the artifacts required for maestro artifact publishing. The `add-build-to-channel` darc command will now trigger a [Build Promotion Pipeline][0] that pushes build assets to the feed that corresponds to the maestro channel that is being updated. We should no longer need to push assets to various NuGet feeds in a separate step. [0]: https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=9654802&view=logs&j=ba23343f-f710-5af9-782d-5bd26b102304&t=aec38b44-8611-5dd5-3900-5feff4e06d1b --------- Co-authored-by: Alex Soto <alex@alexsoto.me>
diff --git a/dotnet/package/common.csproj b/dotnet/package/common.csproj
@@ -53,7 +53,6 @@
   <!-- it overwrites packageType -->
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.SharedFramework.Sdk" Version="$(MicrosoftDotNetBuildTasksFeedPackageVersion)" Condition="'$(PackageType)' != 'Template'" />
   <UsingTask TaskName="CreateFrameworkListFile" AssemblyFile="$(DotNetSharedFrameworkTaskFile)" Condition="'$(PackageType)' != 'Template'" />
-  <UsingTask TaskName="GenerateBuildManifest" AssemblyFile="$(_MicrosoftDotNetBuildTasksFeedTaskDir)Microsoft.DotNet.Build.Tasks.Feed.dll" />
 
   <Target Name="_GenerateFrameworkListFile" Condition=" '$(_CreateFrameworkList)' == 'true' Or '$(_CreateRuntimeList)' == 'true' ">
     <!-- https://github.com/dotnet/runtime/blob/0647ec314948904319da5eb15e9931f7c85ed1e2/src/installer/pkg/projects/Directory.Build.targets#L281 -->
@@ -113,15 +112,26 @@
     </Content>
   </ItemGroup>
 
-  <Target Name="GenerateBuildAssetRegistryManifest" >
+  <!-- https://github.com/dotnet/arcade/blob/efc3da96e5ac110513e92ebd9ef87c73f44d8540/Documentation/DependencyFlowOnboardingWithoutArcade.md -->
+  <Target Name="PushManifestToBuildAssetRegistry" >
+    <PropertyGroup>
+      <ArtifactsLogDir>$(BarManifestOutputPath)</ArtifactsLogDir>
+      <AssetManifestFileName>Assets.xml</AssetManifestFileName>
+      <AssetManifestPath>$(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName)</AssetManifestPath>
+    </PropertyGroup>
+
+    <Error Condition="Exists($(AssetManifestPath))" Text="The manifest file '$(AssetManifestPath)' already exists." />
+
     <ItemGroup>
-      <BuildArtifacts Include="$(NupkgPath)\Microsoft*.$(_PlatformName).*.nupkg" />
+      <ItemsToPush Include="$(NupkgPath)\*.nupkg" />
     </ItemGroup>
 
-    <Error Condition="'@(BuildArtifacts)' == ''" Text="No packages to create manifest from." />
+    <Error Condition="'@(ItemsToPush)' == ''" Text="No packages to push." />
+
+    <Message Text="Publishing %(ItemsToPush.Identity)" Importance="normal" />
 
     <ItemGroup>
-      <ManifestBuildData Include="InitialAssetsLocation=https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
+      <ManifestBuildData Include="InitialAssetsLocation=" />
       <ManifestBuildData Include="AzureDevOpsBuildId=$(BUILD_BUILDID)" />
       <ManifestBuildData Include="AzureDevOpsBuildDefinitionId=$(SYSTEM_DEFINITIONID)" />
       <ManifestBuildData Include="AzureDevOpsProject=$(SYSTEM_TEAMPROJECT)" />
@@ -130,31 +140,25 @@
       <ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
     </ItemGroup>
 
-    <GenerateBuildManifest
-        Artifacts="@(BuildArtifacts)"
-        OutputPath="$(BarManifestOutputPath)\AssetManifest.xml"
-        BuildId="$(BUILD_BUILDNUMBER)"
-        BuildData="@(ManifestBuildData)"
-        RepoUri="$(BUILD_REPOSITORY_URI)"
-        RepoBranch="$(BUILD_SOURCEBRANCH)"
-        RepoCommit="$(BUILD_SOURCEVERSION)"
+    <PushToAzureDevOpsArtifacts
+        ItemsToPush="@(ItemsToPush)"
+        ManifestBuildData="@(ManifestBuildData)"
+        ManifestRepoUri="$(BUILD_REPOSITORY_NAME)"
+        ManifestBranch="$(BUILD_SOURCEBRANCH)"
+        ManifestBuildId="$(BUILD_BUILDNUMBER)"
+        ManifestCommit="$(BUILD_SOURCEVERSION)"
+        AssetManifestPath="$(AssetManifestPath)"
         PublishingVersion="3" />
-  </Target>
-
-  <Target Name="PushManifestToBuildAssetRegistry" >
-    <PropertyGroup>
-      <VersionPrefix>1.0.0</VersionPrefix>
-    </PropertyGroup>
 
     <MSBuild
         Targets="Restore"
         Projects="$(PkgMicrosoft_DotNet_Arcade_Sdk)\tools\SdkTasks\PublishBuildAssets.proj"
-        Properties="Configuration=$(Configuration);RepoRoot=$(_RepositoryPath);VersionPrefix=$(VersionPrefix);NuGetPackageRoot=$(_RepositoryPath)\packages\"
+        Properties="Configuration=$(Configuration);RepoRoot=$(_RepositoryPath);VersionPrefix=$(_PackageVersion);NuGetPackageRoot=$(_RepositoryPath)\packages\"
     />
 
     <MSBuild
         Projects="$(PkgMicrosoft_DotNet_Arcade_Sdk)\tools\SdkTasks\PublishBuildAssets.proj"
-        Properties="Configuration=$(Configuration);RepoRoot=$(_RepositoryPath);VersionPrefix=$(VersionPrefix);ManifestsPath=$(BarManifestOutputPath);MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com;NuGetPackageRoot=$(_RepositoryPath)\packages\"
+        Properties="Configuration=$(Configuration);RepoRoot=$(_RepositoryPath);VersionPrefix=$(_PackageVersion);ManifestsPath=$(ArtifactsLogDir)AssetManifest;MaestroApiEndpoint=https://maestro.dot.net;NuGetPackageRoot=$(_RepositoryPath)\packages\"
     />
   </Target>
 
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -70,9 +70,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21212.6">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.24225.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>db49d790a4bfa977a9ed7436bf2aa242cefae45e</Sha>
+      <Sha>67d23f4ba1813b315e7e33c71d18b63475f5c5f8</Sha>
     </Dependency>
     <Dependency Name="Microsoft.TemplateEngine.Tasks" Version="7.0.100-alpha.1.21601.1">
       <Uri>https://github.com/dotnet/templating</Uri>
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -5,7 +5,7 @@
     <MicrosoftDotnetSdkInternalPackageVersion>8.0.107-servicing.24281.6</MicrosoftDotnetSdkInternalPackageVersion>
     <MicrosoftNETILLinkTasksPackageVersion>8.0.5</MicrosoftNETILLinkTasksPackageVersion>
     <MicrosoftNETILLinkPackageVersion>8.0.0-rtm.23524.7</MicrosoftNETILLinkPackageVersion>
-    <MicrosoftDotNetBuildTasksFeedPackageVersion>6.0.0-beta.21212.6</MicrosoftDotNetBuildTasksFeedPackageVersion>
+    <MicrosoftDotNetBuildTasksFeedPackageVersion>8.0.0-beta.24225.1</MicrosoftDotNetBuildTasksFeedPackageVersion>
     <MicrosoftNETCoreAppRefPackageVersion>8.0.5</MicrosoftNETCoreAppRefPackageVersion>
     <MicrosoftNETWorkloadEmscriptenCurrentManifest80100TransportVersion>8.0.0-rtm.23511.3</MicrosoftNETWorkloadEmscriptenCurrentManifest80100TransportVersion>
     <MicrosoftNETRuntimeMonoTargetsSdkPackageVersion>8.0.0</MicrosoftNETRuntimeMonoTargetsSdkPackageVersion>
diff --git a/tools/devops/automation/post-build-pipeline.yml b/tools/devops/automation/post-build-pipeline.yml
@@ -1,5 +1,6 @@
 # YAML pipeline for post build operations. 
 # This pipeline will trigger automatically after successful completion of the `prepare_release` stage against the specified branches.
+# NOTE: This pipeline is now deprecated, maestro publishing has been moved to the "Prepare .NET Release" stage
 
 trigger: none
 pr: none
@@ -33,11 +34,9 @@ resources:
     source: xamarin-macios-ci
     trigger:
       branches:
-      - main
       - release/*
       - net7.0
       - net8.0
-      - net9.0
       - release-test/* # this is for testing the release pipeline on branches without GitHub's branch protection (so we can automate tests without requiring commits to go through pull requests, etc.).
       stages:
       - prepare_release
@@ -111,15 +110,15 @@ jobs:
   # Iterate over each build we pushed and add it to the default channel.
   - powershell: |
       Write-Host "Processing platforms: '$Env:CONFIGURATION_DOTNET_PLATFORMS'"
-      $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+      $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
       $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
       $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
       $dotnetPlatforms = "$Env:CONFIGURATION_DOTNET_PLATFORMS".Split(' ', [StringSplitOptions]::RemoveEmptyEntries)
       & dotnet tool update microsoft.dotnet.darc --version "$darcVersion" --add-source "$arcadeServicesSource" --tool-path $(Agent.ToolsDirectory)\darc -v n
       foreach ($platform in $dotnetPlatforms) {
         $envVar = "ENVIRONMENT_REDIRECT_$($platform.ToUpper())_BARBUILDID"
         $barBuildId = [Environment]::GetEnvironmentVariable($envVar)
-        & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $barBuildId --publishing-infra-version 3 --skip-assets-publishing --password $(MaestroAccessToken) --azdev-pat $(publishing-dnceng-devdiv-code-r-build-re)
+        & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $barBuildId --publishing-infra-version 3 --password $(MaestroAccessToken) --azdev-pat $(publishing-dnceng-devdiv-code-r-build-re)
         Write-Host "Added $($platform) bar build id: $($barBuildId)"
       }
     displayName: Add builds to default darc channel
diff --git a/tools/devops/automation/templates/build/build-nugets.yml b/tools/devops/automation/templates/build/build-nugets.yml
diff --git a/tools/devops/automation/templates/build/build-pkgs.yml b/tools/devops/automation/templates/build/build-pkgs.yml
@@ -71,7 +71,10 @@ steps:
       timeoutInMinutes: 180
 
     # build nugets
-    - template: build-nugets.yml
+    - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/build-nugets.sh
+      displayName: 'Build Nugets'
+      condition: and(succeeded(), contains(variables['configuration.BuildNugets'], 'True'), ne(variables['ENABLE_DOTNET'], ''))
+      timeoutInMinutes: 180
 
     - bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/generate-workload-rollback.sh
       name: workload_file
diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml
@@ -56,6 +56,10 @@ parameters:
   type: boolean
   default: true # default to true until otherwhise
 
+- name: pushNugetsToMaestro
+  type: boolean
+  default: true
+
 - name: isPR
   type: boolean
 
@@ -438,6 +442,7 @@ stages:
         repositoryAlias: ${{ parameters.repositoryAlias }}
         commit: ${{ parameters.commit }}
         pushNugets: ${{ parameters.pushNugets }}
+        pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }}
 
 - stage: funnel
   displayName: '${{ parameters.stageDisplayNamePrefix }}Collect signed artifacts'
diff --git a/tools/devops/automation/templates/pipelines/build-pipeline.yml b/tools/devops/automation/templates/pipelines/build-pipeline.yml
@@ -67,6 +67,14 @@ parameters:
   type: boolean
   default: false # only to be used when testing the CI and we do not need a signed pkg
 
+- name: pushNugets
+  type: boolean
+  default: true
+
+- name: pushNugetsToMaestro
+  type: boolean
+  default: true
+
 - name: isPR
   displayName: Is PR build
   type: boolean
@@ -241,6 +249,8 @@ stages:
     enableAPIDiff: ${{ parameters.enableAPIDiff }}
     forceInsertion: ${{ parameters.forceInsertion }}
     skipESRP: ${{ parameters.skipESRP }}
+    pushNugets: ${{ parameters.pushNugets }}
+    pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }}
     ${{ if ne(length(parameters.testConfigurations), 0)}}:
       testConfigurations: ${{ parameters.testConfigurations }}
     deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }}
diff --git a/tools/devops/automation/templates/release/vs-insertion-prep.yml b/tools/devops/automation/templates/release/vs-insertion-prep.yml
@@ -21,6 +21,10 @@ parameters:
   type: boolean
   default: true
 
+- name: pushNugetsToMaestro
+  type: boolean
+  default: true
+
 - name: stageDisplayNamePrefix
   type: string
   default: ''
@@ -92,7 +96,18 @@ stages:
       dependsOn: nuget_convert
       condition: "ne(stageDependencies.configure_build.configure.outputs['configure_platforms.ENABLE_DOTNET'],'')"
       variables:
-        skipNugetSecurityAnalysis: true
+      - name: skipNugetSecurityAnalysis
+        value: true
+      - name: INCLUDE_DOTNET_IOS
+        value: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.INCLUDE_DOTNET_IOS'] ]
+      - name: INCLUDE_DOTNET_MACCATALYST
+        value: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.INCLUDE_DOTNET_MACCATALYST'] ]
+      - name: INCLUDE_DOTNET_MACOS
+        value: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.INCLUDE_DOTNET_MACOS'] ]
+      - name: INCLUDE_DOTNET_TVOS
+        value: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.INCLUDE_DOTNET_TVOS'] ]
+      - ${{ if eq(parameters.isPR, false) }}:
+        - group: Publish-Build-Assets
       pool:
         vmImage: windows-latest
       steps:
@@ -103,29 +118,13 @@ stages:
           patterns: |
             *.nupkg
 
-      - task: NuGetCommand@2
-        displayName: Publish Nugets to dotnet8
-        inputs:
-          command: push
-          packagesToPush: '$(Build.SourcesDirectory)/package/*.nupkg'
-          nuGetFeedType: external
-          publishFeedCredentials: dnceng-dotnet8
-
       - task: DownloadPipelineArtifact@2
         inputs:
           artifactName: '${{ parameters.uploadPrefix }}vs-msi-nugets'
           downloadPath: '$(Build.SourcesDirectory)/${{ parameters.uploadPrefix}}vs-msi-nugets'
           patterns: |
             *.nupkg
 
-      - task: NuGetCommand@2
-        displayName: Publish VS MSI Nugets to dotnet8
-        inputs:
-          command: push
-          packagesToPush: '$(Build.SourcesDirectory)/${{ parameters.uploadPrefix }}vs-msi-nugets/*.nupkg'
-          nuGetFeedType: external
-          publishFeedCredentials: dnceng-dotnet8
-
       - pwsh: |
           mkdir $(Build.SourcesDirectory)/nugets-blob
           cp $(Build.SourcesDirectory)/package/* $(Build.SourcesDirectory)/nugets-blob
@@ -168,3 +167,48 @@ stages:
             inputs:
               artifactName: ${{ parameters.uploadPrefix }}vsdrop-multitarget-signed
               downloadPath: $(Build.StagingDirectory)\$(MultiTargetVSDropCommitStatusName)
+
+      - script: make -C $(Build.SourcesDirectory)/dotnet version-props
+        displayName: make version props
+
+      - powershell: |
+          $varMap = @{ "INCLUDE_DOTNET_IOS" = "iOS"; "INCLUDE_DOTNET_MACCATALYST" = "MacCatalyst"; "INCLUDE_DOTNET_MACOS" = "macOS"; "INCLUDE_DOTNET_TVOS" = "tvOS" }
+          foreach ($varName in $varMap.Keys) {
+              if ([Environment]::GetEnvironmentVariable($varName)) {
+                  Write-Host "##vso[task.setvariable variable=MaestroProjectPlatformName]$($varMap[$varName])"
+                  exit 0;
+              }
+          }
+        displayName: Set maestro project variable
+
+      - task: DotNetCoreCLI@2
+        displayName: generate and publish BAR manifest
+        inputs:
+          projects: $(Build.SourcesDirectory)/dotnet/package/Microsoft.$(MaestroProjectPlatformName).Ref/package.csproj
+          arguments: >-
+            -t:PushManifestToBuildAssetRegistry
+            -p:BuildAssetRegistryToken=$(MaestroAccessToken)
+            -p:NupkgPath=$(Build.SourcesDirectory)/nugets-blob
+            -bl:$(Build.ArtifactStagingDirectory)/maestro-binlogs/generate-bar-manifest.binlog
+          workingDirectory: $(Build.SourcesDirectory)\..
+        condition: and(succeeded(), eq('${{ parameters.pushNugetsToMaestro }}', 'true'))
+
+      - powershell: |
+          $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
+          $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
+          $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+          & dotnet tool update microsoft.dotnet.darc --version "$darcVersion" --add-source "$arcadeServicesSource" --tool-path $(Agent.ToolsDirectory)\darc -v n
+          & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $(BARBuildId) --publishing-infra-version 3 --password $(MaestroAccessToken) --azdev-pat $(publishing-dnceng-devdiv-code-r-build-re)
+        displayName: Add builds to default darc channel
+        # We can't use the global.json located in the root of our repo, because makes it required to use the exact .NET version we're referencing in our eng/Versions.Details.xml file.
+        # So in order to not use it, we set the working directory to the parent directory of xamarin-macios.
+        workingDirectory: $(Build.SourcesDirectory)\..
+        condition: and(succeeded(), eq('${{ parameters.pushNugetsToMaestro }}', 'true'))
+
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish Artifact: maestro-binlogs'
+        inputs:
+          targetPath: $(Build.ArtifactStagingDirectory)/maestro-binlogs
+          artifactName: ${{ parameters.uploadPrefix }}maestro-binlogs-$(System.JobAttempt)
+        condition: and(succeededOrFailed(), eq('${{ parameters.pushNugetsToMaestro }}', 'true'))
+        continueOnError: true
