From 09f38bed3bc565268d3bce432157d3e2a930c0d2 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Fri, 8 Sep 2023 18:28:03 +0200 Subject: [PATCH 01/10] [mono][infra] Fix mono-aot-cross build for linux-arm64 using biarch image (#91019) * Use host rootfs when TARGET_BUILD_ARCH matches --- .../common/templates/pipeline-with-resources.yml | 3 ++- src/mono/mono.proj | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index edba5767b612c..d8638e56b865d 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -15,8 +15,9 @@ resources: ROOTFS_DIR: /crossrootfs/armv6 - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64 env: + ROOTFS_HOST_DIR: /crossrootfs/x64 ROOTFS_DIR: /crossrootfs/arm64 - container: linux_musl_x64 diff --git a/src/mono/mono.proj b/src/mono/mono.proj index 3122cdf153cd1..a643e5a049e79 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -810,6 +810,16 @@ + + + $(ROOTFS_HOST_DIR) + + @@ -859,7 +869,7 @@ <_MonoSkipInitCompiler Condition="'$(CrossBuild)' == 'true'">false <_MonoAotCrossOffsetsCommand Condition="'$(MonoUseCrossTool)' == 'true'">$(PythonCmd) $(MonoProjectRoot)mono/tools/offsets-tool/offsets-tool.py @(MonoAotCrossOffsetsToolParams, ' ') <_MonoAotCMakeConfigureCommand>cmake @(MonoAOTCMakeArgs, ' ') $(MonoCMakeExtraArgs) "$(MonoProjectRoot.TrimEnd('\/'))" - <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' != 'true' and '$(HostOS)' != 'windows'">sh -c 'build_arch="$(_CompilerTargetArch)" compiler="$(MonoCCompiler)" . "$(RepositoryEngineeringCommonDir)native/init-compiler.sh" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand)' + <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' != 'true' and '$(HostOS)' != 'windows'">sh -c 'build_arch="$(_CompilerTargetArch)" ROOTFS_DIR="$(MonoCrossDir)" compiler="$(MonoCCompiler)" . "$(RepositoryEngineeringCommonDir)native/init-compiler.sh" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand)' <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' == 'true' and '$(HostOS)' != 'windows'">$(_MonoAOTCCOption) $(_MonoAOTCXXOption) @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand) <_MonoAotCMakeConfigureCommand Condition="'$(HostOS)' == 'windows'">call "$(RepositoryEngineeringDir)native\init-vs-env.cmd" $(_CompilerTargetArch) && cd /D "$(MonoObjCrossDir)" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand) <_MonoAotCMakeBuildCommand>cmake --build . --target install --config $(Configuration) From 36455ca630a561b5862dba89505e95182ef668b3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 27 Sep 2023 10:45:54 -0700 Subject: [PATCH 02/10] Don't build libraries native packages in the PGO leg (#92729) --- eng/pipelines/runtime-official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d84f414f4657c..548b89c7ae1c0 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -601,7 +601,7 @@ extends: - windows_arm64 - linux_arm64 jobParameters: - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO postBuildSteps: From 8dcdff4c51acb7d5d0a7dc4b598a3f759036a065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Mon, 20 Nov 2023 19:25:48 +0100 Subject: [PATCH 03/10] Reenable OneLocBuild in runtime-official.yml (#95014) --- eng/pipelines/runtime-official.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 548b89c7ae1c0..336c06c79e58d 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -41,13 +41,12 @@ extends: # Localization build # - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0') }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - MirrorRepo: runtime - MirrorBranch: release/8.0 - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-RUNTIME' + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + MirrorRepo: runtime + MirrorBranch: main + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-RUNTIME' # # Source Index Build From 76ea33eef52b2dcd1d4840f43140a3035b28907a Mon Sep 17 00:00:00 2001 From: Adeel Mujahid <3840695+am11@users.noreply.github.com> Date: Tue, 16 Jan 2024 23:51:07 +0200 Subject: [PATCH 04/10] Move FreeBSD CI leg to CBL-Mariner and v13 (#97038) --- eng/pipelines/common/templates/pipeline-with-resources.yml | 2 +- src/coreclr/debug/daccess/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index d8638e56b865d..d117043057dc6 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -96,7 +96,7 @@ resources: ROOTFS_DIR: /crossrootfs/x64 - container: freebsd_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-freebsd-12 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13 env: ROOTFS_DIR: /crossrootfs/x64 diff --git a/src/coreclr/debug/daccess/CMakeLists.txt b/src/coreclr/debug/daccess/CMakeLists.txt index 9a867c078019d..5332e957c9eca 100644 --- a/src/coreclr/debug/daccess/CMakeLists.txt +++ b/src/coreclr/debug/daccess/CMakeLists.txt @@ -53,7 +53,7 @@ if(CLR_CMAKE_HOST_FREEBSD OR CLR_CMAKE_HOST_NETBSD OR CLR_CMAKE_HOST_SUNOS) DEPENDS coreclr VERBATIM COMMAND_EXPAND_LISTS - COMMAND ${CLR_DIR}/pal/tools/gen-dactable-rva.sh ${args} + COMMAND ${CMAKE_COMMAND} -E env NM=${CMAKE_NM} ${CLR_DIR}/pal/tools/gen-dactable-rva.sh ${args} COMMENT Generating ${GENERATED_INCLUDE_DIR}/dactablerva.h ) From 78f8f00d71250ac60bb5b15440935a09ddcd489b Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 16 Feb 2024 19:06:19 -0800 Subject: [PATCH 05/10] Use Ubuntu-22.04-based Tizen image (#98589) --- eng/pipelines/common/templates/pipeline-with-resources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index d117043057dc6..1db3c3b866f86 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -101,7 +101,7 @@ resources: ROOTFS_DIR: /crossrootfs/x64 - container: tizen_armel - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-armel-tizen + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen env: ROOTFS_DIR: /crossrootfs/armel From 9b44bd5e2d51eb501cb2775f25b0b9d4da306a11 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Mon, 1 Apr 2024 15:48:15 -0700 Subject: [PATCH 06/10] Start moving over to official templates (#99433) * Start moving over to official templates * Parameterize runtime-inner-loop * Use default false for isOfficialBuild in pipeline-with-resources * Respond to PR comments * Code review comments * Use default template args --- eng/pipelines/common/global-build-job.yml | 30 ++- .../templates/pipeline-with-resources.yml | 232 +++++++++--------- .../templates/publish-build-artifacts.yml | 22 ++ .../templates/publish-pipeline-artifacts.yml | 17 ++ .../templates/runtimes/build-test-job.yml | 18 +- .../common/templates/runtimes/xplat-job.yml | 3 +- .../common/templates/template1es.yml | 31 +++ .../common/templates/templateDispatch.yml | 13 + .../common/templates/templatePublic.yml | 21 ++ eng/pipelines/common/upload-artifact-step.yml | 15 +- .../upload-intermediate-artifacts-step.yml | 14 +- eng/pipelines/common/xplat-setup.yml | 15 +- eng/pipelines/global-build.yml | 1 + .../mono/templates/generate-offsets.yml | 30 ++- .../mono/templates/workloads-build.yml | 18 +- .../jobs/prepare-signed-artifacts.yml | 17 +- eng/pipelines/official/stages/publish.yml | 6 +- eng/pipelines/runtime-official.yml | 31 ++- eng/pipelines/runtime.yml | 2 + 19 files changed, 352 insertions(+), 184 deletions(-) create mode 100644 eng/pipelines/common/templates/publish-build-artifacts.yml create mode 100644 eng/pipelines/common/templates/publish-pipeline-artifacts.yml create mode 100644 eng/pipelines/common/templates/template1es.yml create mode 100644 eng/pipelines/common/templates/templateDispatch.yml create mode 100644 eng/pipelines/common/templates/templatePublic.yml diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 4b06818b1408c..86cea9fbd98a2 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -30,9 +30,10 @@ parameters: extraVariablesTemplates: [] isManualCodeQLBuild: false preBuildSteps: [] + templatePath: 'templates' jobs: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/${{ parameters.templatePath }}/job/job.yml parameters: ${{ if eq(parameters.hostedOs, '') }}: name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} @@ -141,6 +142,7 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} + steps: - ${{ if eq(parameters.osGroup, 'windows') }}: - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml @@ -181,7 +183,7 @@ jobs: path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - ${{ if eq(parameters.isSourceBuild, true) }}: - - template: /eng/common/templates/steps/source-build.yml + - template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml parameters: platform: baseOS: ${{ parameters.baseOS }} @@ -285,14 +287,16 @@ jobs: displayName: Collect vslogs on exit condition: always() - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' - PublishLocation: Container - ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} - ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: ${{ parameters.isOfficialBuild }} + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' + PublishLocation: Container + ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + continueOnError: true + condition: always() diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index 1db3c3b866f86..f5fd4abbd25bf 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -1,114 +1,124 @@ parameters: - name: stages type: stageList - -resources: - containers: - - container: linux_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm - env: - ROOTFS_DIR: /crossrootfs/arm - - - container: linux_armv6 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10 - env: - ROOTFS_DIR: /crossrootfs/armv6 - - - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64 - env: - ROOTFS_HOST_DIR: /crossrootfs/x64 - ROOTFS_DIR: /crossrootfs/arm64 - - - container: linux_musl_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: linux_musl_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine - env: - ROOTFS_DIR: /crossrootfs/arm - - - container: linux_musl_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine - env: - ROOTFS_DIR: /crossrootfs/arm64 - - # This container contains all required toolsets to build for Android and for Linux with bionic libc. - - container: linux_bionic - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 - - # This container contains all required toolsets to build for Android as well as tooling to build docker images. - - container: android_docker - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker - - - container: linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: linux_x86 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86 - env: - ROOTFS_DIR: /crossrootfs/x86 - - - container: linux_x64_dev_innerloop - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - - # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. - - container: SourceBuild_centos_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. - - container: SourceBuild_linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build - - - container: linux_s390x - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x - env: - ROOTFS_DIR: /crossrootfs/s390x - - - container: linux_ppc64le - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le - env: - ROOTFS_DIR: /crossrootfs/ppc64le - - - container: linux_riscv64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64 - env: - ROOTFS_DIR: /crossrootfs/riscv64 - - - container: debian-12-gcc13-amd64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 - - - container: linux_x64_llvmaot - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - - container: browser_wasm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: wasi_wasm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: freebsd_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: tizen_armel - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen - env: - ROOTFS_DIR: /crossrootfs/armel - - - container: debpkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - - container: rpmpkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - -stages: ${{ parameters.stages }} + - name: isOfficialBuild + type: boolean + default: false + +extends: + template: templateDispatch.yml + parameters: + ${{ if parameters.isOfficialBuild }}: + templatePath: template1es.yml + ${{ else }}: + templatePath: templatePublic.yml + + stages: ${{ parameters.stages }} + + containers: + linux_arm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + env: + ROOTFS_DIR: /crossrootfs/arm + + linux_armv6: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10 + env: + ROOTFS_DIR: /crossrootfs/armv6 + + linux_arm64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64 + env: + ROOTFS_HOST_DIR: /crossrootfs/x64 + ROOTFS_DIR: /crossrootfs/arm64 + + linux_musl_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine + env: + ROOTFS_DIR: /crossrootfs/x64 + + linux_musl_arm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine + env: + ROOTFS_DIR: /crossrootfs/arm + + linux_musl_arm64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine + env: + ROOTFS_DIR: /crossrootfs/arm64 + + # This container contains all required toolsets to build for Android and for Linux with bionic libc. + linux_bionic: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 + + # This container contains all required toolsets to build for Android as well as tooling to build docker images. + android_docker: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker + + linux_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 + env: + ROOTFS_DIR: /crossrootfs/x64 + + linux_x86: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86 + env: + ROOTFS_DIR: /crossrootfs/x86 + + linux_x64_dev_innerloop: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + + # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. + SourceBuild_centos_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. + SourceBuild_linux_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build + + linux_s390x: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x + env: + ROOTFS_DIR: /crossrootfs/s390x + + linux_ppc64le: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le + env: + ROOTFS_DIR: /crossrootfs/ppc64le + + linux_riscv64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64 + env: + ROOTFS_DIR: /crossrootfs/riscv64 + + debian-12-gcc13-amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 + + linux_x64_llvmaot: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + browser_wasm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 + env: + ROOTFS_DIR: /crossrootfs/x64 + + wasi_wasm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 + env: + ROOTFS_DIR: /crossrootfs/x64 + + freebsd_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13 + env: + ROOTFS_DIR: /crossrootfs/x64 + + tizen_armel: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen + env: + ROOTFS_DIR: /crossrootfs/armel + + debpkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + + rpmpkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm \ No newline at end of file diff --git a/eng/pipelines/common/templates/publish-build-artifacts.yml b/eng/pipelines/common/templates/publish-build-artifacts.yml new file mode 100644 index 0000000000000..b9b263c361f89 --- /dev/null +++ b/eng/pipelines/common/templates/publish-build-artifacts.yml @@ -0,0 +1,22 @@ +parameters: + - name: isOfficialBuild + type: boolean + - name: displayName + type: string + - name: inputs + type: object + - name: condition + type: string + default: '' + +steps: + - ${{ if parameters.isOfficialBuild }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + condition: ${{ parameters.condition }} + - ${{ else }}: + - task: PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/publish-pipeline-artifacts.yml b/eng/pipelines/common/templates/publish-pipeline-artifacts.yml new file mode 100644 index 0000000000000..81f292ec5528c --- /dev/null +++ b/eng/pipelines/common/templates/publish-pipeline-artifacts.yml @@ -0,0 +1,17 @@ +parameters: +- name: displayName + type: string +- name: inputs + type: object +- name: isOfficialBuild + type: boolean + +steps: + - ${{ if parameters.isOfficialBuild }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index e249e8ac922a7..2809d1487700f 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -14,6 +14,7 @@ parameters: dependsOn: [] dependOnEvaluatePaths: false crossBuild: false + isOfficialBuild: false ### Build managed test components (native components are getting built as part ### of the product build job). @@ -142,12 +143,13 @@ jobs: artifactName: $(microsoftNetSdkIlArtifactName) displayName: 'Microsoft.NET.Sdk.IL package' - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + ArtifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' + continueOnError: true + condition: always() diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index 23e74c70e57ac..625d88d63d3e9 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -20,11 +20,12 @@ parameters: enableMicrobuild: '' gatherAssetManifests: false disableComponentGovernance: false + templatePath: 'templates' variables: {} ## any extra variables to add to the defaults defined below jobs: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/${{ parameters.templatePath }}/job/job.yml parameters: name: ${{ parameters.name }} diff --git a/eng/pipelines/common/templates/template1es.yml b/eng/pipelines/common/templates/template1es.yml new file mode 100644 index 0000000000000..0770e37d6bd0e --- /dev/null +++ b/eng/pipelines/common/templates/template1es.yml @@ -0,0 +1,31 @@ + + +parameters: + - name: templatePath + type: string + default: 'templates-official' + - name: stages + type: stageList + - name: containers + type: object + + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + containers: + ${{ parameters.containers }} + + stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/templateDispatch.yml b/eng/pipelines/common/templates/templateDispatch.yml new file mode 100644 index 0000000000000..1860af47aeeff --- /dev/null +++ b/eng/pipelines/common/templates/templateDispatch.yml @@ -0,0 +1,13 @@ +parameters: + - name: templatePath + type: string + - name: stages + type: stageList + - name: containers + type: object + +extends: + template: ${{ parameters.templatePath }} + parameters: + stages: ${{ parameters.stages }} + containers: ${{ parameters.containers }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/templatePublic.yml b/eng/pipelines/common/templates/templatePublic.yml new file mode 100644 index 0000000000000..cd7c02720167d --- /dev/null +++ b/eng/pipelines/common/templates/templatePublic.yml @@ -0,0 +1,21 @@ + +parameters: + - name: templatePath + type: string + default: 'templates' + - name: stages + type: stageList + - name: containers + type: object + +resources: + containers: + - ${{ each container_pair in parameters.containers }}: + - ${{ if container_pair.value.image }}: + - container: ${{ container_pair.key }} + ${{ each pair in container_pair.value }}: + ${{ if notIn(pair.key, 'tenantId', 'identityType', 'registry') }}: + ${{ pair.key }}: ${{ pair.value }} + + +stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/common/upload-artifact-step.yml b/eng/pipelines/common/upload-artifact-step.yml index 249da066c7aae..d4091a7cc192f 100644 --- a/eng/pipelines/common/upload-artifact-step.yml +++ b/eng/pipelines/common/upload-artifact-step.yml @@ -7,6 +7,7 @@ parameters: artifactName: '' displayName: '' condition: succeeded() + isOfficialBuild: false steps: # Zip Artifact @@ -20,9 +21,11 @@ steps: includeRootFolder: ${{ parameters.includeRootFolder }} condition: ${{ parameters.condition }} - - task: PublishBuildArtifacts@1 - displayName: 'Publish ${{ parameters.displayName }}' - inputs: - pathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} - artifactName: ${{ parameters.artifactName }} - condition: ${{ parameters.condition }} + - template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: ${{ parameters.isOfficialBuild }} + displayName: 'Publish ${{ parameters.displayName }}' + inputs: + PathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} + artifactName: ${{ parameters.artifactName }} + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/pipelines/common/upload-intermediate-artifacts-step.yml b/eng/pipelines/common/upload-intermediate-artifacts-step.yml index bde6c61a0a04a..da9b1ef0b627f 100644 --- a/eng/pipelines/common/upload-intermediate-artifacts-step.yml +++ b/eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -25,9 +25,11 @@ steps: TargetFolder: '$(Build.StagingDirectory)/IntermediateArtifacts/${{ parameters.name }}' CleanTargetFolder: true -- task: PublishBuildArtifacts@1 - displayName: Publish intermediate artifacts - inputs: - pathToPublish: '$(Build.StagingDirectory)/IntermediateArtifacts' - artifactName: IntermediateArtifacts - artifactType: container +- template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: true + displayName: Publish intermediate artifacts + inputs: + PathtoPublish: '$(Build.StagingDirectory)/IntermediateArtifacts' + ArtifactName: IntermediateArtifacts + ArtifactType: container diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 675a267920186..794a23bb218b5 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -22,7 +22,7 @@ jobs: dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests', 'dotnet-linker-tests', 'runtime-dev-innerloop', 'runtime-coreclr superpmi-replay', 'runtime-coreclr superpmi-diffs')) }} variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/${{ coalesce(parameters.jobParameters.templatePath, 'templates') }}/variables/pool-providers.yml # Disable component governance in our CI builds. These builds are not shipping nor # are they a service. Also the component governance jobs issue lots of inconsequential # warnings and errors into our build timelines that make it hard to track down @@ -168,12 +168,19 @@ jobs: # Official Build Linux Pool ${{ if and(or(in(parameters.osGroup, 'linux', 'freebsd', 'android', 'tizen'), eq(parameters.jobParameters.hostedOs, 'linux')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + demands: ImageOverride -equals 1es-ubuntu-2204 + os: linux - # OSX Build Pool (we don't have on-prem OSX BuildPool). - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: + # OSX Public Build Pool (we don't have on-prem OSX BuildPool). + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), eq(variables['System.TeamProject'], 'public')) }}: vmImage: 'macos-12' + # OSX Internal Pool + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), ne(variables['System.TeamProject'], 'public')) }}: + name: "Azure Pipelines" + vmImage: 'macOS-12' + os: macOS + # Official Build Windows Pool ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 6f80e6ad10452..82e4cac024958 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -30,6 +30,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: false stages: - stage: Build jobs: diff --git a/eng/pipelines/mono/templates/generate-offsets.yml b/eng/pipelines/mono/templates/generate-offsets.yml index c68adfe67a9e6..909ca9a99c54d 100644 --- a/eng/pipelines/mono/templates/generate-offsets.yml +++ b/eng/pipelines/mono/templates/generate-offsets.yml @@ -9,11 +9,13 @@ parameters: pool: '' condition: true isOfficialBuild: false + templatePath: 'templates' ### Product build jobs: - template: xplat-pipeline-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} osGroup: ${{ parameters.osGroup }} osSubGroup: ${{ parameters.osSubGroup }} @@ -76,17 +78,21 @@ jobs: contents: '**/offsets-*.h' targetFolder: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles/' - - task: PublishPipelineArtifact@1 - displayName: Upload offset files - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - artifactName: 'Mono_Offsets_$(osGroup)$(osSubGroup)' + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Upload offset files + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' + artifactName: 'Mono_Offsets_$(osGroup)$(osSubGroup)' # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' + continueOnError: true + condition: always() diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index 4a50ed665e24e..a10bf343fa4f1 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -12,11 +12,13 @@ parameters: runtimeVariant: '' testGroup: '' timeoutInMinutes: '' + templatePath: 'templates' variables: {} jobs: - template: xplat-pipeline-job.yml parameters: + templatePath: ${{ parameters.templatePath }} archType: ${{ parameters.archType }} buildConfig: ${{ parameters.buildConfig }} container: ${{ parameters.container }} @@ -92,13 +94,15 @@ jobs: name: workloads # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' + continueOnError: true + condition: always() # Delete wixpdb files before they are uploaded to artifacts - task: DeleteFiles@1 diff --git a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml index 908f2b64c71c2..24fd2df48d74b 100644 --- a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -20,6 +20,14 @@ jobs: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)\BuildLogs' + artifactName: ${{ parameters.logArtifactName }} + steps: - checkout: self clean: true @@ -65,11 +73,4 @@ jobs: **/*.binlog TargetFolder: '$(Build.StagingDirectory)\BuildLogs' continueOnError: true - condition: succeededOrFailed() - - - task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)\BuildLogs' - artifactName: ${{ parameters.logArtifactName }} - condition: succeededOrFailed() + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml index d23afa7003b25..9553baae305f2 100644 --- a/eng/pipelines/official/stages/publish.yml +++ b/eng/pipelines/official/stages/publish.yml @@ -7,7 +7,7 @@ stages: - stage: PrepareForPublish displayName: Prepare for Publish variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml jobs: # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. - template: /eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -15,7 +15,7 @@ stages: PublishRidAgnosticPackagesFromPlatform: ${{ parameters.PublishRidAgnosticPackagesFromPlatform }} # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - - template: /eng/common/templates/job/publish-build-assets.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: publishUsingPipelines: true publishAssetsImmediately: true @@ -26,7 +26,7 @@ stages: symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' # Stages-based publishing entry point -- template: /eng/common/templates/post-build/post-build.yml +- template: /eng/common/templates-official/post-build/post-build.yml parameters: publishingInfraVersion: ${{ parameters.publishingInfraVersion }} validateDependsOn: diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 336c06c79e58d..1bc58b0afa143 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -33,6 +33,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: true stages: - stage: Build jobs: @@ -41,7 +42,7 @@ extends: # Localization build # - - template: /eng/common/templates/job/onelocbuild.yml + - template: /eng/common/templates-official/job/onelocbuild.yml parameters: MirrorRepo: runtime MirrorBranch: main @@ -52,7 +53,7 @@ extends: # Source Index Build # - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - - template: /eng/common/templates/job/source-index-stage1.yml + - template: /eng/common/templates-official/job/source-index-stage1.yml parameters: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true @@ -69,6 +70,7 @@ extends: - windows_x64 - windows_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -104,6 +106,7 @@ extends: platforms: - windows_x86 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -137,6 +140,7 @@ extends: - osx_arm64 - osx_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -196,6 +200,7 @@ extends: - linux_musl_arm - linux_musl_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.corelib+clr.nativecorelib+clr.tools+clr.aot+clr.packages+libs+host+packs -c $(_BuildConfig) nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -208,12 +213,12 @@ extends: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) Contents: libcoreclr.so TargetFolder: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts/$(osGroup)$(osSubgroup).$(archType).$(_BuildConfig)/$(crossDacHostArch) - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishBuildArtifacts@1 displayName: Publish runtime for CrossDac inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts + PathtoPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts PublishLocation: Container - artifactName: CoreCLRCrossDacArtifacts + ArtifactName: CoreCLRCrossDacArtifacts # Create RPMs and DEBs - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml parameters: @@ -249,6 +254,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -319,6 +325,7 @@ extends: - linux_bionic_arm64 - linux_bionic_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c $(_BuildConfig) /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true nameSuffix: NativeAOT isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -362,6 +369,7 @@ extends: - windows_x86 # - windows_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -379,6 +387,7 @@ extends: - browser_wasm - wasi_wasm jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -395,6 +404,7 @@ extends: platforms: - browser_wasm jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoWasmBuildVariant=multithread /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -417,6 +427,7 @@ extends: - ios_arm64 - maccatalyst_x64 jobParameters: + templatePath: 'templates-official' isOfficialBuild: ${{ variables.isOfficialBuild }} # @@ -433,6 +444,7 @@ extends: - linux_arm64 - linux_musl_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -457,6 +469,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -482,6 +495,7 @@ extends: - osx_x64 - osx_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser+tvos+ios+maccatalyst /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -525,6 +539,7 @@ extends: buildConfig: release runtimeFlavor: mono jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false nameSuffix: Mono_LLVMJIT @@ -539,6 +554,7 @@ extends: buildConfig: release runtimeFlavor: mono jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT @@ -559,6 +575,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_AllConfigurations isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -578,7 +595,9 @@ extends: platforms: - SourceBuild_linux_x64 jobParameters: + templatePath: 'templates-official' nameSuffix: PortableSourceBuild + isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml parameters: @@ -600,6 +619,7 @@ extends: - windows_arm64 - linux_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO @@ -619,6 +639,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 dependsOn: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 767b3dc9ba424..8098e999f2c0c 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -52,6 +52,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: false stages: - stage: Build jobs: @@ -428,6 +429,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/mono/templates/generate-offsets.yml + templatePath: 'templates' buildConfig: release platforms: - android_x64 From 9237f2f3fd87216e3afa702fa786d9d738a0211b Mon Sep 17 00:00:00 2001 From: Eduardo Velarde Date: Thu, 16 May 2024 11:08:02 -0700 Subject: [PATCH 07/10] Move linux_x86, tizen_armel, and freebsd_x64 jobs to global-build.yml --- eng/pipelines/global-build.yml | 94 ++++++++++++++++++++++++++++++++++ eng/pipelines/runtime.yml | 45 ---------------- 2 files changed, 94 insertions(+), 45 deletions(-) diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 82e4cac024958..8b6291cd0db0b 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -178,3 +178,97 @@ extends: timeoutInMinutes: 95 condition: eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) + + # + # Build CoreCLR as a non-portable build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - tizen_armel + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NonPortable + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR with no R2R + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_x86 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NoR2R + buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR release + # Always as they are needed by Installer and we always build and test the Installer. + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: release + platforms: + - freebsd_x64 + jobParameters: + testGroup: innerloop + # Mono/runtimetests also need this, but skip for wasm + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - freebsd_x64 + jobParameters: + testScope: innerloop + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/installer/jobs/build-job.yml + buildConfig: Release + platforms: + - freebsd_x64 + jobParameters: + liveRuntimeBuildConfig: release + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runOnlyIfDependenciesSucceeded: true + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 8098e999f2c0c..8f001c4903003 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -156,7 +156,6 @@ extends: - windows_x64 - windows_x86 - windows_arm64 - - freebsd_x64 jobParameters: testGroup: innerloop # Mono/runtimetests also need this, but skip for wasm @@ -188,48 +187,6 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_jit.containsChange'], true), eq(variables['isRollingBuild'], true))) - # - # Build CoreCLR with no R2R - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: checked - runtimeFlavor: coreclr - platforms: - - linux_x86 - jobParameters: - testScope: innerloop - nameSuffix: CoreCLR_NoR2R - buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) - timeoutInMinutes: 120 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build CoreCLR as a non-portable build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: checked - runtimeFlavor: coreclr - platforms: - - tizen_armel - jobParameters: - testScope: innerloop - nameSuffix: CoreCLR_NonPortable - buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false - timeoutInMinutes: 120 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) - # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed @@ -1061,7 +1018,6 @@ extends: - osx_arm64 - osx_x64 - windows_x64 - - freebsd_x64 jobParameters: testScope: innerloop condition: @@ -1185,7 +1141,6 @@ extends: - linux_arm64 - linux_musl_x64 - windows_x64 - - freebsd_x64 jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} From b2336550327d88cb665b059ff7d6b3b4c6f7ce89 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Thu, 21 Mar 2024 17:53:55 +0100 Subject: [PATCH 08/10] Update thunktemplates.S (#100066) --- src/coreclr/vm/arm/thunktemplates.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/arm/thunktemplates.S b/src/coreclr/vm/arm/thunktemplates.S index 0686bb2ed4b73..8744c8ebb6326 100644 --- a/src/coreclr/vm/arm/thunktemplates.S +++ b/src/coreclr/vm/arm/thunktemplates.S @@ -11,7 +11,7 @@ PAGE_SIZE = 4096 -#define DATA_SLOT(stub, field) stub##Code + PAGE_SIZE + stub##Data__##field +#define DATA_SLOT(stub, field) . - (. - stub##Code) + PAGE_SIZE + stub##Data__##field LEAF_ENTRY StubPrecodeCode ldr r12, DATA_SLOT(StubPrecode, MethodDesc) From 775aed03e3388b63733e7cfbd7f414896e1c9e15 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 10 Apr 2024 10:19:02 -0700 Subject: [PATCH 09/10] Import pool providers (#100851) Import pool-providers in common variables. This should allow all stages access to the pool provider variables. --- eng/pipelines/common/variables.yml | 7 +++++++ eng/pipelines/runtime-official.yml | 2 ++ 2 files changed, 9 insertions(+) diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index 2a757f82572ad..075dea178ab62 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -1,3 +1,8 @@ +parameters: + - name: templatePath + type: string + default: 'templates' + variables: # These values enable longer delays, configurable number of retries, and special understanding of TCP hang-up @@ -54,3 +59,5 @@ variables: eq(variables['isRollingBuild'], true))) ] - template: /eng/pipelines/common/perf-variables.yml + +- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml \ No newline at end of file diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 1bc58b0afa143..7cadf6a800f67 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -23,6 +23,8 @@ pr: none variables: - template: /eng/pipelines/common/variables.yml + parameters: + templatePath: 'templates-official' - template: /eng/pipelines/common/internal-variables.yml parameters: teamName: dotnet-core-acquisition From 0f15d9e1d4c8861b55c0c03e26f66f23f193318d Mon Sep 17 00:00:00 2001 From: Eduardo Manuel Velarde Polar Date: Tue, 28 May 2024 16:02:49 -0700 Subject: [PATCH 10/10] Pass templatePath to eng/pipelines/common/templates/runtimes/xplat-job.yml --- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 2 ++ eng/pipelines/mono/templates/xplat-pipeline-job.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index b6c57be0c7fba..82d6346a60d7a 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -11,6 +11,7 @@ parameters: liveLibrariesBuildConfig: '' strategy: '' pool: '' + templatePath: 'templates' # arcade-specific parameters condition: true @@ -28,6 +29,7 @@ parameters: jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 1ca84d9caac1b..67b43722e0c57 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -12,6 +12,7 @@ parameters: pool: '' runtimeVariant: '' liveRuntimeBuildConfig: 'release' + templatePath: 'templates' # arcade-specific parameters condition: true @@ -28,6 +29,7 @@ parameters: jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }}