From cc7af6ceb0ad8437651213954e12ed120ca1ffff Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Thu, 21 Mar 2024 15:32:06 -0700
Subject: [PATCH 01/11] Rebase the installer build jobs on the global build job
template.
---
eng/pipelines/installer/jobs/build-job.yml | 248 ------------------
.../jobs/steps/upload-job-artifacts.yml | 65 -----
.../{jobs => }/steps/build-linux-package.yml | 3 -
.../installer/steps/upload-job-artifacts.yml | 26 ++
eng/pipelines/runtime-official.yml | 4 +-
eng/pipelines/runtime.yml | 113 +++++++-
6 files changed, 127 insertions(+), 332 deletions(-)
delete mode 100644 eng/pipelines/installer/jobs/build-job.yml
delete mode 100644 eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml
rename eng/pipelines/installer/{jobs => }/steps/build-linux-package.yml (91%)
create mode 100644 eng/pipelines/installer/steps/upload-job-artifacts.yml
diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml
deleted file mode 100644
index 9a444b2bde170d..00000000000000
--- a/eng/pipelines/installer/jobs/build-job.yml
+++ /dev/null
@@ -1,248 +0,0 @@
-parameters:
- buildConfig: ''
- osGroup: ''
- archType: ''
- osSubgroup: ''
- platform: ''
- crossBuild: false
- timeoutInMinutes: 120
- condition: true
- shouldContinueOnError: false
- container: ''
- buildSteps: []
- dependsOn: []
- globalBuildSuffix: ''
- variables: []
- name: ''
- displayName: ''
- runtimeVariant: ''
- pool: ''
-
- # The target names here should match container names in the resources section in our pipelines, like runtime.yml
- packageDistroList:
- - target: debpkg
- packageType: deb
- packagingArgs: /p:BuildDebPackage=true
- - target: rpmpkg
- packageType: rpm
- packagingArgs: /p:BuildRpmPackage=true
-
- isOfficialBuild: false
- buildFullPlatformManifest: false
-
- liveRuntimeBuildConfig: ''
- liveLibrariesBuildConfig: ''
- runtimeFlavor: 'coreclr'
- unifiedArtifactsName: ''
- unifiedBuildNameSuffix: ''
- unifiedBuildConfigOverride: ''
-
-### Product build
-jobs:
-- template: /eng/common/templates/job/job.yml
- parameters:
- buildConfig: ${{ parameters.buildConfig }}
- archType: ${{ parameters.archType }}
- osGroup: ${{ parameters.osGroup }}
- osSubgroup: ${{ parameters.osSubgroup }}
- runtimeVariant: ${{ parameters.runtimeVariant }}
- testGroup: ${{ parameters.testGroup }}
- helixType: 'build/product/'
- enableMicrobuild: true
- pool: ${{ parameters.pool }}
-
- # Compute job name from template parameters
- name: ${{ format('installer_{0}_{1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }}
- displayName: ${{ format('Installer Build and Test {0} {1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }}
-
- # Run all steps in the container.
- # Note that the containers are defined in platform-matrix.yml
- container: ${{ parameters.container }}
-
- timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
-
- crossBuild: ${{ parameters.crossBuild }}
-
- gatherAssetManifests: true
-
- # Component governance does not work on musl machines
- ${{ if eq(parameters.osSubGroup, '_musl') }}:
- disableComponentGovernance: true
-
- variables:
- - ${{ each variable in parameters.variables }}:
- - ${{ variable }}
-
- - name: BuildAction
- value: -test
-
- - name: buildCommandSourcesDirectory
- value: '$(Build.SourcesDirectory)/'
-
- - name: librariesBuildConfigArg
- value: -lc ${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }}
-
- - name: runtimeBuildConfigArg
- value: -rc ${{ coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig) }}
-
- ###
- ### Platform-specific variable setup
- ###
-
- - ${{ if eq(parameters.osGroup, 'windows') }}:
-
- - name: CommonMSBuildArgs
- value: >-
- /p:TargetArchitecture=${{ parameters.archType }}
- /p:PortableBuild=true
- /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
- - name: TargetArchitecture
- value: ${{ parameters.archType }}
-
- - name: BaseJobBuildCommand
- value: >-
- build.cmd -subset host+packs -ci
- $(BuildAction)
- -configuration $(_BuildConfig)
- $(librariesBuildConfigArg)
- $(runtimeBuildConfigArg)
- $(CommonMSBuildArgs)
-
- - ${{ elseif eq(parameters.osGroup, 'osx') }}:
-
- - name: CommonMSBuildArgs
- value: >-
- /p:PortableBuild=true
- /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
- /p:TargetArchitecture=${{ parameters.archType }}
- /p:CrossBuild=${{ parameters.crossBuild }}
-
- - name: BaseJobBuildCommand
- value: >-
- $(Build.SourcesDirectory)/build.sh -subset host+packs -ci
- $(BuildAction)
- -configuration $(_BuildConfig)
- -arch ${{ parameters.archType }}
- $(librariesBuildConfigArg)
- $(runtimeBuildConfigArg)
- $(CommonMSBuildArgs)
-
- - ${{ elseif in(parameters.osGroup, 'linux', 'freebsd') }}:
- - name: CommonMSBuildArgs
- value: >-
- /p:Configuration=$(_BuildConfig)
- /p:TargetOS=${{ parameters.osGroup }}
- /p:TargetArchitecture=${{ parameters.archType }}
- /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
-
- - name: _PortableBuild
- value: ${{ eq(parameters.osSubgroup, '') }}
-
- - ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'linux')) }}:
- - name: _PortableBuild
- value: true
-
- - ${{ if eq(parameters.crossBuild, true) }}:
- - name: ArchArg
- value: -arch ${{ parameters.archType }}
-
- - name: BaseJobBuildCommand
- value: >-
- $(Build.SourcesDirectory)/build.sh -subset host+packs -ci
- $(BuildAction)
- $(librariesBuildConfigArg)
- $(runtimeBuildConfigArg)
- $(ArchArg)
- /p:CrossBuild=${{ parameters.crossBuild }}
- /p:PortableBuild=$(_PortableBuild)
- $(CommonMSBuildArgs)
-
- - name: installersSubsetArg
- value: --subset packs.installers
-
- dependsOn:
- - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}'
- - ${{ parameters.dependsOn }}
- steps:
- - ${{ if eq(parameters.osGroup, 'windows') }}:
- - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml
- - checkout: self
- clean: true
- fetchDepth: $(checkoutFetchDepth)
-
- - ${{ if ne(variables['System.TeamProject'], 'public') }}:
- - ${{ if ne(parameters.osGroup, 'windows') }}:
- - task: Bash@3
- displayName: Setup Private Feeds Credentials
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
- arguments: $(Build.SourcesDirectory)/NuGet.config $Token
- env:
- Token: $(dn-bot-dnceng-artifact-feeds-rw)
- - ${{ else }}:
- - task: PowerShell@2
- displayName: Setup Private Feeds Credentials
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
- arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
- env:
- Token: $(dn-bot-dnceng-artifact-feeds-rw)
-
- - template: /eng/pipelines/common/download-artifact-step.yml
- parameters:
- unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
- artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)'
- artifactName: '${{ parameters.unifiedArtifactsName }}'
- displayName: 'unified artifacts'
-
- - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}:
- - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }}
- displayName: Install Build Dependencies
-
- - script: |
- du -sh $(Build.SourcesDirectory)/*
- df -h
- displayName: Disk Usage before Build
-
- - script: $(BaseJobBuildCommand)
- displayName: Build
- continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }}
-
- - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}:
- - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log
- displayName: Collect vslogs on exit
- condition: always()
-
- - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}:
- - script: |
- du -sh $(Build.SourcesDirectory)/*
- df -h
- displayName: Disk Usage after Build
-
- # Only in glibc leg, we produce RPMs and Debs
- - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'linux_x64'), eq(parameters.platform, 'linux_arm64')), eq(parameters.osSubgroup, ''))}}:
- - ${{ each packageBuild in parameters.packageDistroList }}:
- # This leg's RID matches the build image. Build its distro-dependent packages, as well as
- # the distro-independent installers. (There's no particular reason to build the distro-
- # independent installers on this leg, but we need to do it somewhere.)
- # Currently, linux_arm64 supports 'rpm' type only.
- - ${{ if or(not(eq(parameters.platform, 'linux_arm64')), eq(packageBuild.packageType, 'rpm')) }}:
- - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml
- parameters:
- packageType: ${{ packageBuild.packageType }}
- target: ${{ packageBuild.target }}
- packageStepDescription: Runtime Deps, Runtime, Framework Packs installers
- subsetArg: $(installersSubsetArg)
- packagingArgs: ${{ packageBuild.packagingArgs }}
-
- - template: /eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml
- parameters:
- name: ${{ coalesce(parameters.name, parameters.platform) }}
- runtimeFlavor: ${{ parameters.runtimeFlavor }}
- runtimeVariant: ${{ parameters.runtimeVariant }}
-
- - ${{ if ne(parameters.osGroup, 'windows') }}:
- - script: set -x && df -h
- displayName: Check remaining storage space
- condition: always()
- continueOnError: true
diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml
deleted file mode 100644
index 8cdf6e8fb77787..00000000000000
--- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml
+++ /dev/null
@@ -1,65 +0,0 @@
-parameters:
- name: ''
-
-steps:
-- task: PublishTestResults@2
- displayName: Publish Test Results
- inputs:
- testResultsFormat: 'VSTest'
- testResultsFiles: '*.trx'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
- mergeTestResults: true
- testRunTitle: Installer-${{ parameters.name }}-$(_BuildConfig)
- continueOnError: true
- condition: always()
-
-# Upload binaries and symbols on failure to allow debugging issues
-- task: CopyFiles@2
- displayName: Prepare binaries to publish
- inputs:
- SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin'
- Contents: |
- */corehost/**
- */corehost_test/**
- TargetFolder: '$(Build.StagingDirectory)/Binaries'
- continueOnError: true
- condition: failed()
-
-- task: ArchiveFiles@2
- displayName: Zip binaries
- inputs:
- rootFolderOrFile: '$(Build.StagingDirectory)/Binaries'
- archiveFile: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)'
- archiveType: $(archiveType)
- tarCompression: $(tarCompression)
- includeRootFolder: false
- continueOnError: true
- condition: failed()
-
-- task: PublishBuildArtifacts@1
- displayName: Publish binaries
- inputs:
- pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)'
- artifactName: Installer-Binaries-${{ parameters.name }}-$(_BuildConfig)
- continueOnError: true
- condition: failed()
-
-- task: CopyFiles@2
- displayName: Prepare BuildLogs staging directory
- inputs:
- SourceFolder: '$(Build.SourcesDirectory)'
- Contents: |
- **/*.log
- **/*.binlog
- TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
- CleanTargetFolder: true
- continueOnError: true
- condition: always()
-
-- task: PublishPipelineArtifact@1
- displayName: Publish BuildLogs
- inputs:
- targetPath: '$(Build.StagingDirectory)/BuildLogs'
- artifactName: Installer-Logs_Attempt$(System.JobAttempt)-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig)
- continueOnError: true
- condition: always()
diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml
similarity index 91%
rename from eng/pipelines/installer/jobs/steps/build-linux-package.yml
rename to eng/pipelines/installer/steps/build-linux-package.yml
index 7f8320b55dfe39..bcc8a9a8e9706c 100644
--- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml
+++ b/eng/pipelines/installer/steps/build-linux-package.yml
@@ -16,9 +16,6 @@ steps:
--ci \
${{ parameters.subsetArg }} \
${{ parameters.packagingArgs }} \
- $(CommonMSBuildArgs) \
- $(librariesBuildConfigArg) \
- $(runtimeBuildConfigArg) \
/bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog
displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }}
target: ${{ parameters.target }}
diff --git a/eng/pipelines/installer/steps/upload-job-artifacts.yml b/eng/pipelines/installer/steps/upload-job-artifacts.yml
new file mode 100644
index 00000000000000..076f989780bbfe
--- /dev/null
+++ b/eng/pipelines/installer/steps/upload-job-artifacts.yml
@@ -0,0 +1,26 @@
+parameters:
+ name: ''
+
+steps:
+# Upload binaries and symbols on failure to allow debugging issues
+- task: CopyFiles@2
+ displayName: Prepare binaries to publish
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin'
+ Contents: |
+ */corehost/**
+ */corehost_test/**
+ TargetFolder: '$(Build.StagingDirectory)/Binaries'
+ continueOnError: true
+ condition: failed()
+
+- template: /eng/pipelines/common/upload-artifact-step.yml
+ parameters:
+ rootFolder: '$(Build.StagingDirectory)/Binaries'
+ includeRootFolder: false
+ archiveType: $(archiveType)
+ archiveExtension: $(archiveExtension)
+ tarCompression: $(tarCompression)
+ artifactName: 'Installer-Binaries-${{ parameters.name }}-$(_BuildConfig)'
+ displayName: 'Binaries'
+ condition: failed()
diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml
index 0cc897455e2e06..1b9c18c6be33cc 100644
--- a/eng/pipelines/runtime-official.yml
+++ b/eng/pipelines/runtime-official.yml
@@ -208,7 +208,7 @@ extends:
PublishLocation: Container
artifactName: CoreCLRCrossDacArtifacts
# Create RPMs and DEBs
- - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml
+ - template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: deb
target: debpkg
@@ -216,7 +216,7 @@ extends:
subsetArg: -s packs.installers
packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true
condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64'))
- - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml
+ - template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: rpm
target: rpmpkg
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 3cce0b5a17499f..d1805acf12629f 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -94,12 +94,34 @@ extends:
buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
platforms:
- osx_arm64
+ jobParameters:
+ nameSuffix: AllSubsets_CoreCLR
+ buildArgs: -s clr+libs+host+packs -rc Release -c Release -lc $(_BuildConfig)
+ timeoutInMinutes: 120
+ condition: >-
+ or(
+ eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
+ eq(variables['isRollingBuild'], true))
+
+ - template: /eng/pipelines/common/platform-matrix.yml
+ parameters:
+ jobTemplate: /eng/pipelines/common/global-build-job.yml
+ buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ platforms:
- linux_arm64
- linux_musl_x64
jobParameters:
nameSuffix: AllSubsets_CoreCLR
buildArgs: -s clr+libs+host+packs -rc Release -c Release -lc $(_BuildConfig)
timeoutInMinutes: 120
+ postBuildSteps:
+ - template: /eng/pipelines/installer/steps/build-linux-package.yml
+ parameters:
+ packageType: rpm
+ target: rpmpkg
+ packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers
+ subsetArg: -s packs.installers
+ packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true
condition: >-
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
@@ -1351,16 +1373,27 @@ extends:
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
- jobTemplate: /eng/pipelines/installer/jobs/build-job.yml
+ jobTemplate: /eng/pipelines/common/global-build-job.yml
buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
platforms:
- windows_x86
jobParameters:
- liveRuntimeBuildConfig: release
- liveLibrariesBuildConfig: Release
- unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release
- unifiedBuildConfigOverride: release
- unifiedBuildNameSuffix: CoreCLR_Libraries
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ dependsOn:
+ - build_$(osGroup)$(osSubgroup)_$(archType)_release_CoreCLR_Libraries
+ preBuildSteps:
+ - template: /eng/pipelines/common/download-artifact-step.yml
+ parameters:
+ artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release
+ artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release
+ unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
+ displayName: 'unified artifacts'
+ enablePublishTestResults: true
+ testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
+ postBuildSteps:
+ - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
+ parameters:
+ name: $(osGroup)$(osSubgroup)_$(archType)
condition:
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
@@ -1368,18 +1401,70 @@ extends:
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
- jobTemplate: /eng/pipelines/installer/jobs/build-job.yml
- buildConfig: Release
+ jobTemplate: /eng/pipelines/common/global-build-job.yml
+ buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
platforms:
- osx_x64
- - linux_x64
- windows_x64
jobParameters:
- liveRuntimeBuildConfig: release
- liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
- unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
- unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }}
- unifiedBuildNameSuffix: CoreCLR_Libraries
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ dependsOn:
+ - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries
+ preBuildSteps:
+ - template: /eng/pipelines/common/download-artifact-step.yml
+ parameters:
+ artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
+ displayName: 'unified artifacts'
+ enablePublishTestResults: true
+ testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
+ postBuildSteps:
+ - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
+ parameters:
+ name: $(osGroup)$(osSubgroup)_$(archType)
+ condition:
+ or(
+ eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
+ eq(variables['isRollingBuild'], true))
+
+ - template: /eng/pipelines/common/platform-matrix.yml
+ parameters:
+ jobTemplate: /eng/pipelines/common/global-build-job.yml
+ buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ platforms:
+ - linux_x64
+ jobParameters:
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ dependsOn:
+ - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries
+ preBuildSteps:
+ - template: /eng/pipelines/common/download-artifact-step.yml
+ parameters:
+ artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
+ displayName: 'unified artifacts'
+ enablePublishTestResults: true
+ testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
+ postBuildSteps:
+ - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
+ parameters:
+ name: $(osGroup)$(osSubgroup)_$(archType)
+ - template: /eng/pipelines/installer/steps/build-linux-package.yml
+ parameters:
+ packageType: deb
+ target: debpkg
+ packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers
+ subsetArg: -s packs.installers
+ packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true
+ - template: /eng/pipelines/installer/steps/build-linux-package.yml
+ parameters:
+ packageType: rpm
+ target: rpmpkg
+ packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers
+ subsetArg: -s packs.installers
+ packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true
condition:
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
From 14ca5f0320c5990a7d5966e22dac40ca6f5bdf46 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Thu, 21 Mar 2024 15:39:44 -0700
Subject: [PATCH 02/11] Rebase build-linux-package on global-build-step now
that it's only used in global-build-job jobs.
---
.../installer/steps/build-linux-package.yml | 18 ++++++++----------
eng/pipelines/runtime-official.yml | 8 ++------
eng/pipelines/runtime.yml | 12 +++---------
3 files changed, 13 insertions(+), 25 deletions(-)
diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml
index bcc8a9a8e9706c..57baa4a74fce9a 100644
--- a/eng/pipelines/installer/steps/build-linux-package.yml
+++ b/eng/pipelines/installer/steps/build-linux-package.yml
@@ -3,7 +3,6 @@ parameters:
target: ''
packageStepDescription: null
packagingArgs: ''
- subsetArg: ''
condition: succeeded()
steps:
@@ -11,15 +10,14 @@ steps:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- task: NuGetAuthenticate@1
target: ${{ parameters.target }}
-- script: |
- $(Build.SourcesDirectory)/build.sh \
- --ci \
- ${{ parameters.subsetArg }} \
- ${{ parameters.packagingArgs }} \
- /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog
- displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }}
- target: ${{ parameters.target }}
- condition: ${{ parameters.condition }}
+
+- template: /eng/pipelines/common/templates/global-build-step.yml
+ parameters:
+ buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog
+ container: ${{ parameters.target }}
+ displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages
+ condition: ${{ parameters.condition }}
+
# Broken symbolic links break the SBOM processing
# We make some symlinks during the installer generation process,
# but they aren't always valid on disk afterwards. Some of our tooling,
diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml
index 1b9c18c6be33cc..44dfe5781c3bd4 100644
--- a/eng/pipelines/runtime-official.yml
+++ b/eng/pipelines/runtime-official.yml
@@ -212,17 +212,13 @@ extends:
parameters:
packageType: deb
target: debpkg
- packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers
- subsetArg: -s packs.installers
- packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true
+ packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true
condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64'))
- template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: rpm
target: rpmpkg
- packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers
- subsetArg: -s packs.installers
- packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true
+ packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
condition: and(succeeded(), eq(variables.osSubgroup, ''), in(variables.archType, 'x64', 'arm64'))
# Upload the results.
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index d1805acf12629f..a17fce1e91a963 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -119,9 +119,7 @@ extends:
parameters:
packageType: rpm
target: rpmpkg
- packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers
- subsetArg: -s packs.installers
- packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true
+ packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
condition: >-
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
@@ -1455,16 +1453,12 @@ extends:
parameters:
packageType: deb
target: debpkg
- packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers
- subsetArg: -s packs.installers
- packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true
+ packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true
- template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: rpm
target: rpmpkg
- packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers
- subsetArg: -s packs.installers
- packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true
+ packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
condition:
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
From 3304e445ab6baeb1b7d07df4e6fee4f99e9d8d32 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Thu, 21 Mar 2024 15:53:19 -0700
Subject: [PATCH 03/11] Template-style variables for dependsOn?
---
eng/pipelines/runtime.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index a17fce1e91a963..2e0bd0c2c4696e 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -1378,7 +1378,7 @@ extends:
jobParameters:
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- - build_$(osGroup)$(osSubgroup)_$(archType)_release_CoreCLR_Libraries
+ - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_release_CoreCLR_Libraries
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
@@ -1407,7 +1407,7 @@ extends:
jobParameters:
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries
+ - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
@@ -1435,7 +1435,7 @@ extends:
jobParameters:
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- - build_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)_CoreCLR_Libraries
+ - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
From 6b499ad7f4dcaf7e0a4d1a154cd8ccf900fafcf6 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Thu, 21 Mar 2024 15:56:23 -0700
Subject: [PATCH 04/11] Expand out the jobs to explicitly state dependencies
---
eng/pipelines/runtime.yml | 33 ++++++++++++++++++++++++++++++---
1 file changed, 30 insertions(+), 3 deletions(-)
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 2e0bd0c2c4696e..d3cca40f274791 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -1378,7 +1378,7 @@ extends:
jobParameters:
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_release_CoreCLR_Libraries
+ - build_windows_x86_release_CoreCLR_Libraries
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
@@ -1403,11 +1403,38 @@ extends:
buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
platforms:
- osx_x64
+ jobParameters:
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ dependsOn:
+ - build_osx_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
+ preBuildSteps:
+ - template: /eng/pipelines/common/download-artifact-step.yml
+ parameters:
+ artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
+ displayName: 'unified artifacts'
+ enablePublishTestResults: true
+ testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
+ postBuildSteps:
+ - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
+ parameters:
+ name: $(osGroup)$(osSubgroup)_$(archType)
+ condition:
+ or(
+ eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
+ eq(variables['isRollingBuild'], true))
+
+ - template: /eng/pipelines/common/platform-matrix.yml
+ parameters:
+ jobTemplate: /eng/pipelines/common/global-build-job.yml
+ buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ platforms:
- windows_x64
jobParameters:
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
+ - build_windows_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
@@ -1435,7 +1462,7 @@ extends:
jobParameters:
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- - build_${{ variables.osGroup }}${{ variables.osSubgroup }}_${{ variables.archType }}_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
+ - build_linux_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
From 359005fbc171e67137571913db02f39924bc7288 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Thu, 21 Mar 2024 16:49:39 -0700
Subject: [PATCH 05/11] Installer build steps aren't cross builds
---
eng/pipelines/common/templates/global-build-step.yml | 3 ++-
eng/pipelines/installer/steps/build-linux-package.yml | 2 ++
eng/pipelines/runtime.yml | 1 +
3 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml
index 6597ac4feaefcd..f732a5e27a4a9c 100644
--- a/eng/pipelines/common/templates/global-build-step.yml
+++ b/eng/pipelines/common/templates/global-build-step.yml
@@ -3,11 +3,12 @@ parameters:
useContinueOnErrorDuringBuild: false
shouldContinueOnError: false
archParameter: $(_archParameter)
+ crossArg: $(crossArg)
displayName: Build product
container: ''
steps:
- - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter)
+ - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter)
displayName: ${{ parameters.displayName }}
${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}:
continueOnError: ${{ parameters.shouldContinueOnError }}
diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml
index 57baa4a74fce9a..443c580e189158 100644
--- a/eng/pipelines/installer/steps/build-linux-package.yml
+++ b/eng/pipelines/installer/steps/build-linux-package.yml
@@ -16,6 +16,8 @@ steps:
buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog
container: ${{ parameters.target }}
displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages
+ # Even for cross-build targets, our installer build steps are not cross-builds
+ crossArg: ''
condition: ${{ parameters.condition }}
# Broken symbolic links break the SBOM processing
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index d3cca40f274791..72849c2e56e411 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -120,6 +120,7 @@ extends:
packageType: rpm
target: rpmpkg
packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
+ condition: and(succeeded(), eq(variables.osSubgroup, ''))
condition: >-
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
From 33e159c58f37778854d2cd3bd384c4eace01ecf9 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Thu, 21 Mar 2024 18:36:01 -0700
Subject: [PATCH 06/11] Forward condition down and fix config
---
eng/pipelines/common/templates/global-build-step.yml | 2 ++
eng/pipelines/runtime.yml | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml
index f732a5e27a4a9c..b88207c516c19b 100644
--- a/eng/pipelines/common/templates/global-build-step.yml
+++ b/eng/pipelines/common/templates/global-build-step.yml
@@ -6,6 +6,7 @@ parameters:
crossArg: $(crossArg)
displayName: Build product
container: ''
+ condition: succeeded()
steps:
- script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter)
@@ -14,3 +15,4 @@ steps:
continueOnError: ${{ parameters.shouldContinueOnError }}
${{ if ne(parameters.container, '') }}:
target: ${{ parameters.container }}
+ condition: ${{ parameters.condition }}
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 72849c2e56e411..4403f1a8c85576 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -119,7 +119,7 @@ extends:
parameters:
packageType: rpm
target: rpmpkg
- packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
+ packagingArgs: -c Release /p:BuildRpmPackage=true
condition: and(succeeded(), eq(variables.osSubgroup, ''))
condition: >-
or(
From 521b84acab3737910a1826178c6c855b702182e9 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Mon, 1 Apr 2024 15:19:00 -0700
Subject: [PATCH 07/11] Fix cross-config problems
---
eng/pipelines/runtime.yml | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 4403f1a8c85576..553e519e3a7320 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -1373,10 +1373,11 @@ extends:
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ buildConfig: release
platforms:
- windows_x86
jobParameters:
+ nameSuffix: Installer_Build_And_Test
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
dependsOn:
- build_windows_x86_release_CoreCLR_Libraries
@@ -1401,11 +1402,12 @@ extends:
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ buildConfig: release
platforms:
- osx_x64
jobParameters:
- buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ nameSuffix: Installer_Build_And_Test
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
dependsOn:
- build_osx_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
@@ -1429,11 +1431,12 @@ extends:
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ buildConfig: release
platforms:
- windows_x64
jobParameters:
- buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ nameSuffix: Installer_Build_And_Test
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
dependsOn:
- build_windows_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
@@ -1457,11 +1460,12 @@ extends:
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+ buildConfig: release
platforms:
- linux_x64
jobParameters:
- buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
+ nameSuffix: Installer_Build_And_Test
+ buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
dependsOn:
- build_linux_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
preBuildSteps:
From 2034fe0d13090968494c04590d7e9941c9ffdc6f Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Mon, 1 Apr 2024 15:35:19 -0700
Subject: [PATCH 08/11] Re-collapse the installer jobs by providing a
dependency model that can calculate a global build job name when we have a
platform.
---
eng/pipelines/common/global-build-job.yml | 14 +++-
.../installer/steps/build-linux-package.yml | 4 +-
eng/pipelines/runtime.yml | 72 +++----------------
3 files changed, 25 insertions(+), 65 deletions(-)
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
index eb83415112e7b7..421aacce1a607f 100644
--- a/eng/pipelines/common/global-build-job.yml
+++ b/eng/pipelines/common/global-build-job.yml
@@ -12,6 +12,13 @@ parameters:
targetRid: ''
timeoutInMinutes: ''
dependsOn: []
+ # The following parameter is used to specify dependencies on other global build for the same platform.
+ # We provide this mechanism to allow for global builds to depend on other global builds and use the multiplexing
+ # that platform-matrix.yml enables.
+ # Each item can have the following properties:
+ # - nameSuffix: The suffix of the job name to depend on.
+ # - buildConfig: The configuration of the job to depend on.
+ dependsOnGlobalBuilds: []
pool: ''
platform: ''
condition: true
@@ -61,8 +68,11 @@ jobs:
workspace:
clean: all
- ${{ if ne(parameters.dependsOn,'') }}:
- dependsOn: ${{ parameters.dependsOn }}
+ ${{ if or(ne(parameters.dependsOn,''), ne(parameters.dependsOnGlobalBuilds,'')) }}:
+ dependsOn:
+ - ${{ insert }} : ${{ parameters.dependsOn }}
+ - ${{ each globalBuild in parameters.dependsOnGlobalBuilds }}:
+ - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, coalesce(globalBuild.buildConfig, parameters.buildConfig), globalBuild.nameSuffix) }}
variables:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml
index 443c580e189158..5c7dcb57ef0604 100644
--- a/eng/pipelines/installer/steps/build-linux-package.yml
+++ b/eng/pipelines/installer/steps/build-linux-package.yml
@@ -7,9 +7,10 @@ parameters:
steps:
## Run NuGet Authentication for each of the side containers
-- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(parameters.target, '')) }}:
- task: NuGetAuthenticate@1
target: ${{ parameters.target }}
+ condition: ${{ parameters.condition }}
- template: /eng/pipelines/common/templates/global-build-step.yml
parameters:
@@ -26,3 +27,4 @@ steps:
# in particular the SBOM tooling, breaks on broken symlinks.
- script: find . -xtype l -delete
displayName: Remove broken symbolic links
+ condition: ${{ parameters.condition }}
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 553e519e3a7320..9816d8b43f81f9 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -1373,14 +1373,15 @@ extends:
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: release
+ buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
platforms:
- windows_x86
jobParameters:
nameSuffix: Installer_Build_And_Test
buildArgs: -s host+packs -c $(_BuildConfig) -lc Release -rc Release -test
- dependsOn:
- - build_windows_x86_release_CoreCLR_Libraries
+ dependsOnGlobalBuilds:
+ - nameSuffix: CoreCLR_Libraries
+ buildConfig: release
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
@@ -1399,75 +1400,20 @@ extends:
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
eq(variables['isRollingBuild'], true))
- - template: /eng/pipelines/common/platform-matrix.yml
- parameters:
- jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: release
- platforms:
- - osx_x64
- jobParameters:
- nameSuffix: Installer_Build_And_Test
- buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
- dependsOn:
- - build_osx_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
- preBuildSteps:
- - template: /eng/pipelines/common/download-artifact-step.yml
- parameters:
- artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
- artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
- unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
- displayName: 'unified artifacts'
- enablePublishTestResults: true
- testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
- postBuildSteps:
- - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
- parameters:
- name: $(osGroup)$(osSubgroup)_$(archType)
- condition:
- or(
- eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
- eq(variables['isRollingBuild'], true))
-
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
buildConfig: release
platforms:
- windows_x64
- jobParameters:
- nameSuffix: Installer_Build_And_Test
- buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
- dependsOn:
- - build_windows_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
- preBuildSteps:
- - template: /eng/pipelines/common/download-artifact-step.yml
- parameters:
- artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
- artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
- unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
- displayName: 'unified artifacts'
- enablePublishTestResults: true
- testRunTitle: Installer-$(osGroup)$(osSubgroup)_$(archType)
- postBuildSteps:
- - template: /eng/pipelines/installer/steps/upload-job-artifacts.yml
- parameters:
- name: $(osGroup)$(osSubgroup)_$(archType)
- condition:
- or(
- eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
- eq(variables['isRollingBuild'], true))
-
- - template: /eng/pipelines/common/platform-matrix.yml
- parameters:
- jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: release
- platforms:
+ - osx_x64
- linux_x64
jobParameters:
nameSuffix: Installer_Build_And_Test
buildArgs: -s host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} -rc Release -test
- dependsOn:
- - build_linux_x64_${{ variables.debugOnPrReleaseOnRolling }}_CoreCLR_Libraries
+ dependsOnGlobalBuilds:
+ - nameSuffix: CoreCLR_Libraries
+ buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
preBuildSteps:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
@@ -1486,11 +1432,13 @@ extends:
packageType: deb
target: debpkg
packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true
+ condition: eq(variables['osGroup'], 'linux')
- template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: rpm
target: rpmpkg
packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
+ condition: eq(variables['osGroup'], 'linux')
condition:
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
From 9e55e61bff67f8bdf04d1399df3606776d588ee9 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Mon, 1 Apr 2024 15:37:48 -0700
Subject: [PATCH 09/11] Try using an each instead of insert.
---
eng/pipelines/common/global-build-job.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
index 421aacce1a607f..9b81acc80d811f 100644
--- a/eng/pipelines/common/global-build-job.yml
+++ b/eng/pipelines/common/global-build-job.yml
@@ -70,7 +70,8 @@ jobs:
${{ if or(ne(parameters.dependsOn,''), ne(parameters.dependsOnGlobalBuilds,'')) }}:
dependsOn:
- - ${{ insert }} : ${{ parameters.dependsOn }}
+ - ${{ each build in parameters.dependsOn }}:
+ - ${{ build }}
- ${{ each globalBuild in parameters.dependsOnGlobalBuilds }}:
- ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, coalesce(globalBuild.buildConfig, parameters.buildConfig), globalBuild.nameSuffix) }}
From 2c62cc27d91b5841d23c04782b33e8ae01c338e5 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jkoritzinsky@gmail.com>
Date: Mon, 1 Apr 2024 18:14:34 -0700
Subject: [PATCH 10/11] Remove docker initialization attempts on non-Windows
installer jobs
---
.../installer/steps/build-linux-package.yml | 43 ++++++++++---------
eng/pipelines/runtime.yml | 2 -
2 files changed, 23 insertions(+), 22 deletions(-)
diff --git a/eng/pipelines/installer/steps/build-linux-package.yml b/eng/pipelines/installer/steps/build-linux-package.yml
index 5c7dcb57ef0604..ef905d6c164657 100644
--- a/eng/pipelines/installer/steps/build-linux-package.yml
+++ b/eng/pipelines/installer/steps/build-linux-package.yml
@@ -1,4 +1,6 @@
parameters:
+ osGroup: ''
+ osSubgroup: ''
packageType: null
target: ''
packageStepDescription: null
@@ -6,25 +8,26 @@ parameters:
condition: succeeded()
steps:
-## Run NuGet Authentication for each of the side containers
-- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(parameters.target, '')) }}:
- - task: NuGetAuthenticate@1
- target: ${{ parameters.target }}
- condition: ${{ parameters.condition }}
+- ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '')) }}:
+ ## Run NuGet Authentication for each of the side containers
+ - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(parameters.target, '')) }}:
+ - task: NuGetAuthenticate@1
+ target: ${{ parameters.target }}
+ condition: ${{ parameters.condition }}
-- template: /eng/pipelines/common/templates/global-build-step.yml
- parameters:
- buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog
- container: ${{ parameters.target }}
- displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages
- # Even for cross-build targets, our installer build steps are not cross-builds
- crossArg: ''
- condition: ${{ parameters.condition }}
+ - template: /eng/pipelines/common/templates/global-build-step.yml
+ parameters:
+ buildArgs: -s packs.installers ${{ parameters.packagingArgs }} /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog
+ container: ${{ parameters.target }}
+ displayName: Package Runtime Deps, Runtime, Framework Packs - ${{ parameters.packageType }} packages
+ # Even for cross-build targets, our installer build steps are not cross-builds
+ crossArg: ''
+ condition: ${{ parameters.condition }}
-# Broken symbolic links break the SBOM processing
-# We make some symlinks during the installer generation process,
-# but they aren't always valid on disk afterwards. Some of our tooling,
-# in particular the SBOM tooling, breaks on broken symlinks.
-- script: find . -xtype l -delete
- displayName: Remove broken symbolic links
- condition: ${{ parameters.condition }}
+ # Broken symbolic links break the SBOM processing
+ # We make some symlinks during the installer generation process,
+ # but they aren't always valid on disk afterwards. Some of our tooling,
+ # in particular the SBOM tooling, breaks on broken symlinks.
+ - script: find . -xtype l -delete
+ displayName: Remove broken symbolic links
+ condition: ${{ parameters.condition }}
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 9816d8b43f81f9..e31b72f7c5836b 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -1432,13 +1432,11 @@ extends:
packageType: deb
target: debpkg
packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true
- condition: eq(variables['osGroup'], 'linux')
- template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: rpm
target: rpmpkg
packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
- condition: eq(variables['osGroup'], 'linux')
condition:
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),
From 2822dde9ad735e3510a0efe99672f43e97aacd73 Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Tue, 2 Apr 2024 10:25:44 -0700
Subject: [PATCH 11/11] Fix unified artifacts path
---
eng/pipelines/runtime.yml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index e31b72f7c5836b..6c35d9573d8c89 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -119,7 +119,7 @@ extends:
parameters:
packageType: rpm
target: rpmpkg
- packagingArgs: -c Release /p:BuildRpmPackage=true
+ packagingArgs: -c Release -lc $(_BuildConfig) /p:BuildRpmPackage=true
condition: and(succeeded(), eq(variables.osSubgroup, ''))
condition: >-
or(
@@ -1386,7 +1386,7 @@ extends:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release
- artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release
+ artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release$(archiveExtension)
unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
displayName: 'unified artifacts'
enablePublishTestResults: true
@@ -1418,7 +1418,7 @@ extends:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
- artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)
+ artifactFileName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling)$(archiveExtension)
unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
displayName: 'unified artifacts'
enablePublishTestResults: true
@@ -1431,12 +1431,12 @@ extends:
parameters:
packageType: deb
target: debpkg
- packagingArgs: -c $(_BuildConfig) /p:BuildDebPackage=true
+ packagingArgs: -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} /p:BuildDebPackage=true
- template: /eng/pipelines/installer/steps/build-linux-package.yml
parameters:
packageType: rpm
target: rpmpkg
- packagingArgs: -c $(_BuildConfig) /p:BuildRpmPackage=true
+ packagingArgs: -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} /p:BuildRpmPackage=true
condition:
or(
eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true),