Skip to content

Commit

Permalink
[release/8.0-staging] Backport 1ES templates (#102681)
Browse files Browse the repository at this point in the history
* [mono][infra] Fix mono-aot-cross build for linux-arm64 using biarch image (#91019)

* Use host rootfs when TARGET_BUILD_ARCH matches

* Don't build libraries native packages in the PGO leg (#92729)

* Reenable OneLocBuild in runtime-official.yml (#95014)

* Move FreeBSD CI leg to CBL-Mariner and v13 (#97038)

* Use Ubuntu-22.04-based Tizen image (#98589)

* Start moving over to official templates (#99433)

* Start moving over to official templates

* Parameterize runtime-inner-loop

* Use default false for isOfficialBuild in pipeline-with-resources

* Respond to PR comments

* Code review comments

* Use default template args

* Move linux_x86, tizen_armel, and freebsd_x64 jobs to global-build.yml

* Update thunktemplates.S (#100066)

* Import pool providers (#100851)

Import pool-providers in common variables. This should allow all stages access to the pool provider variables.

* Pass templatePath to eng/pipelines/common/templates/runtimes/xplat-job.yml

---------

Co-authored-by: Milos Kotlar <kotlarmilos@gmail.com>
Co-authored-by: Jeremy Koritzinsky <jekoritz@microsoft.com>
Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>
Co-authored-by: Andy Gocke <angocke@microsoft.com>
Co-authored-by: Filip Navara <filip.navara@gmail.com>
Co-authored-by: Eduardo Manuel Velarde Polar <evelardepola@microsoft.com>
  • Loading branch information
8 people authored May 29, 2024
1 parent 7b08fa8 commit 32400f6
Show file tree
Hide file tree
Showing 25 changed files with 478 additions and 238 deletions.
30 changes: 17 additions & 13 deletions eng/pipelines/common/global-build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ parameters:
extraVariablesTemplates: []
isManualCodeQLBuild: false
preBuildSteps: []
templatePath: 'templates'

jobs:
- template: /eng/common/templates/job/job.yml
- template: /eng/common/${{ parameters.templatePath }}/job/job.yml
parameters:
${{ if eq(parameters.hostedOs, '') }}:
name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }}
Expand Down Expand Up @@ -141,6 +142,7 @@ jobs:

- ${{ each variable in parameters.variables }}:
- ${{ variable }}

steps:
- ${{ if eq(parameters.osGroup, 'windows') }}:
- template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml
Expand Down Expand Up @@ -181,7 +183,7 @@ jobs:
path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'

- ${{ if eq(parameters.isSourceBuild, true) }}:
- template: /eng/common/templates/steps/source-build.yml
- template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml
parameters:
platform:
baseOS: ${{ parameters.baseOS }}
Expand Down Expand Up @@ -285,14 +287,16 @@ jobs:
displayName: Collect vslogs on exit
condition: always()

- task: PublishBuildArtifacts@1
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/'
PublishLocation: Container
${{ if notin(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
${{ if in(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
continueOnError: true
condition: always()
- template: /eng/pipelines/common/templates/publish-build-artifacts.yml
parameters:
isOfficialBuild: ${{ parameters.isOfficialBuild }}
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/'
PublishLocation: Container
${{ if notin(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
${{ if in(parameters.osGroup, 'browser', 'wasi') }}:
ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }}
continueOnError: true
condition: always()
231 changes: 121 additions & 110 deletions eng/pipelines/common/templates/pipeline-with-resources.yml
Original file line number Diff line number Diff line change
@@ -1,113 +1,124 @@
parameters:
- name: stages
type: stageList

resources:
containers:
- container: linux_arm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
env:
ROOTFS_DIR: /crossrootfs/arm

- container: linux_armv6
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10
env:
ROOTFS_DIR: /crossrootfs/armv6

- container: linux_arm64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64
env:
ROOTFS_DIR: /crossrootfs/arm64

- container: linux_musl_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
env:
ROOTFS_DIR: /crossrootfs/x64

- container: linux_musl_arm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
env:
ROOTFS_DIR: /crossrootfs/arm

- container: linux_musl_arm64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
env:
ROOTFS_DIR: /crossrootfs/arm64

# This container contains all required toolsets to build for Android and for Linux with bionic libc.
- container: linux_bionic
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64

# This container contains all required toolsets to build for Android as well as tooling to build docker images.
- container: android_docker
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker

- container: linux_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
env:
ROOTFS_DIR: /crossrootfs/x64

- container: linux_x86
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86
env:
ROOTFS_DIR: /crossrootfs/x86

- container: linux_x64_dev_innerloop
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04

# We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9.
- container: SourceBuild_centos_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9

# AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8.
- container: SourceBuild_linux_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build

- container: linux_s390x
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x
env:
ROOTFS_DIR: /crossrootfs/s390x

- container: linux_ppc64le
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le
env:
ROOTFS_DIR: /crossrootfs/ppc64le

- container: linux_riscv64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64
env:
ROOTFS_DIR: /crossrootfs/riscv64

- container: debian-12-gcc13-amd64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64

- container: linux_x64_llvmaot
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9

- container: browser_wasm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
env:
ROOTFS_DIR: /crossrootfs/x64

- container: wasi_wasm
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
env:
ROOTFS_DIR: /crossrootfs/x64

- container: freebsd_x64
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-freebsd-12
env:
ROOTFS_DIR: /crossrootfs/x64

- container: tizen_armel
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-armel-tizen
env:
ROOTFS_DIR: /crossrootfs/armel

- container: debpkg
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg

- container: rpmpkg
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm

stages: ${{ parameters.stages }}
- name: isOfficialBuild
type: boolean
default: false

extends:
template: templateDispatch.yml
parameters:
${{ if parameters.isOfficialBuild }}:
templatePath: template1es.yml
${{ else }}:
templatePath: templatePublic.yml

stages: ${{ parameters.stages }}

containers:
linux_arm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
env:
ROOTFS_DIR: /crossrootfs/arm

linux_armv6:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10
env:
ROOTFS_DIR: /crossrootfs/armv6

linux_arm64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64
env:
ROOTFS_HOST_DIR: /crossrootfs/x64
ROOTFS_DIR: /crossrootfs/arm64

linux_musl_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
env:
ROOTFS_DIR: /crossrootfs/x64

linux_musl_arm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
env:
ROOTFS_DIR: /crossrootfs/arm

linux_musl_arm64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
env:
ROOTFS_DIR: /crossrootfs/arm64

# This container contains all required toolsets to build for Android and for Linux with bionic libc.
linux_bionic:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64

# This container contains all required toolsets to build for Android as well as tooling to build docker images.
android_docker:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker

linux_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
env:
ROOTFS_DIR: /crossrootfs/x64

linux_x86:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86
env:
ROOTFS_DIR: /crossrootfs/x86

linux_x64_dev_innerloop:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04

# We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9.
SourceBuild_centos_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9

# AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8.
SourceBuild_linux_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build

linux_s390x:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x
env:
ROOTFS_DIR: /crossrootfs/s390x

linux_ppc64le:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le
env:
ROOTFS_DIR: /crossrootfs/ppc64le

linux_riscv64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64
env:
ROOTFS_DIR: /crossrootfs/riscv64

debian-12-gcc13-amd64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64

linux_x64_llvmaot:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9

browser_wasm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
env:
ROOTFS_DIR: /crossrootfs/x64

wasi_wasm:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
env:
ROOTFS_DIR: /crossrootfs/x64

freebsd_x64:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13
env:
ROOTFS_DIR: /crossrootfs/x64

tizen_armel:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen
env:
ROOTFS_DIR: /crossrootfs/armel

debpkg:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg

rpmpkg:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm
22 changes: 22 additions & 0 deletions eng/pipelines/common/templates/publish-build-artifacts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
parameters:
- name: isOfficialBuild
type: boolean
- name: displayName
type: string
- name: inputs
type: object
- name: condition
type: string
default: ''

steps:
- ${{ if parameters.isOfficialBuild }}:
- task: 1ES.PublishBuildArtifacts@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
condition: ${{ parameters.condition }}
- ${{ else }}:
- task: PublishBuildArtifacts@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
condition: ${{ parameters.condition }}
17 changes: 17 additions & 0 deletions eng/pipelines/common/templates/publish-pipeline-artifacts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
parameters:
- name: displayName
type: string
- name: inputs
type: object
- name: isOfficialBuild
type: boolean

steps:
- ${{ if parameters.isOfficialBuild }}:
- task: 1ES.PublishPipelineArtifact@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
- ${{ else }}:
- task: PublishPipelineArtifact@1
displayName: ${{ parameters.displayName }}
inputs: ${{ parameters.inputs }}
18 changes: 10 additions & 8 deletions eng/pipelines/common/templates/runtimes/build-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ parameters:
dependsOn: []
dependOnEvaluatePaths: false
crossBuild: false
isOfficialBuild: false

### Build managed test components (native components are getting built as part
### of the product build job).
Expand Down Expand Up @@ -142,12 +143,13 @@ jobs:
artifactName: $(microsoftNetSdkIlArtifactName)
displayName: 'Microsoft.NET.Sdk.IL package'


# Publish Logs
- task: PublishPipelineArtifact@1
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}'
continueOnError: true
condition: always()
- template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml
parameters:
displayName: Publish Logs
isOfficialBuild: ${{ parameters.isOfficialBuild }}
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
ArtifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}'
continueOnError: true
condition: always()
3 changes: 2 additions & 1 deletion eng/pipelines/common/templates/runtimes/xplat-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ parameters:
enableMicrobuild: ''
gatherAssetManifests: false
disableComponentGovernance: false
templatePath: 'templates'

variables: {} ## any extra variables to add to the defaults defined below

jobs:
- template: /eng/common/templates/job/job.yml
- template: /eng/common/${{ parameters.templatePath }}/job/job.yml
parameters:

name: ${{ parameters.name }}
Expand Down
Loading

0 comments on commit 32400f6

Please sign in to comment.