Build on CBL-mariner host with rootfs (#84148)

This changes the linux builds to use the new CBL-mariner build images. All of these builds are now cross-builds with a rootfs (including x64, and x64 musl). - The new images intentionally don't have binutils, so our build jobs have been updated to use llvm-based tools. This includes lld when we run nativeaot over our tests. This leads to a slight size regression, I believe due to ILCompiler not supporting `--gc-sections` with lld. - This change turns off PGO optimization because it was hitting a compiler crash when consuming and old version of profile data produced by clang9 instrumented builds (https://bugzilla.redhat.com/show_bug.cgi?id=1827282). The intention is to quickly flow the change into our optimization repo, then collect and ingest the new data so we can turn optimization back on. - The Mono LLVMAot test build is kept on the old build images, because they run the aot compiler which depends on binutils. We can probably move it to the same plan as the other legs, but I'm not familiar with the context here. cc @akoeplinger. We would at least need to fix `AS_NAME` here: https://github.com/dotnet/runtime/blob/1d2cd206f40306cbf06f3e2676e7cfa301077285/src/mono/mono/mini/aot-compiler.c#L13030 - Mono arm64 product build is kept on the old images due to #84503. - Includes a temporary revert of d80a584 which requires LLVM 13. - Sets arch for mono build, because our source-built LLVM doesn't target armv7a by default. Possibly related to https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/b7363248b115339c4fb838fcd3ae43671eedae0a - Allow text relocations. The lld defaults differ from ld, which will add text relocations on demand by default. See https://maskray.me/blog/2020-12-19-lld-and-gnu-linker-incompatibilities --------- Co-authored-by: Jan Vorlicek <janvorli@microsoft.com> Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com> Co-authored-by: Jeremy Koritzinsky <jkoritzinsky@gmail.com> Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
dotnet · Apr 12, 2023 · 887c043 · 887c043
1 parent 3be7ac1
commit 887c043
Show file tree

Hide file tree

Showing 40 changed files with 281 additions and 115 deletions.
diff --git a/eng/Subsets.props b/eng/Subsets.props
@@ -249,8 +249,8 @@
       Category="clr" />
   </ItemGroup>
 
-  <!-- Build the CoreCLR cross-arch tools when we're doing a cross-architecture build and either we're building any CoreCLR native tools for platforms CoreCLR fully supports or when someone explicitly requests them -->
-  <ItemGroup Condition="(('$(ClrRuntimeBuildSubsets)' != '' and '$(PrimaryRuntimeFlavor)' == 'CoreCLR' and '$(TargetsMobile)' != 'true') or $(_subset.Contains('+clr.crossarchtools+'))) and '$(BuildArchitecture)' != '$(TargetArchitecture)'">
+  <!-- Build the CoreCLR cross tools when we're doing a cross build and either we're building any CoreCLR native tools for platforms CoreCLR fully supports or when someone explicitly requests them -->
+  <ItemGroup Condition="(('$(ClrRuntimeBuildSubsets)' != '' and '$(PrimaryRuntimeFlavor)' == 'CoreCLR' and '$(TargetsMobile)' != 'true') or $(_subset.Contains('+clr.crossarchtools+'))) and ('$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)')">
     <ProjectToBuild
       Include="$(CoreClrProjectRoot)runtime.proj"
       AdditionalProperties="%(AdditionalProperties);
@@ -325,7 +325,7 @@
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\ILCompiler.csproj" Category="clr" Condition="'$(NativeAotSupported)' == 'true'" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)nativeaot\BuildIntegration\BuildIntegration.proj" Category="clr" Condition="'$(NativeAotSupported)' == 'true'" />
 
-    <ProjectToBuild Condition="'$(NativeAotSupported)' == 'true' and '$(TargetArchitecture)' != '$(BuildArchitecture)'" Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\ILCompiler_crossarch.csproj" Category="clr" />
+    <ProjectToBuild Condition="'$(NativeAotSupported)' == 'true' and ('$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)')" Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\ILCompiler_crossarch.csproj" Category="clr" />
     <ProjectToBuild Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)'" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_crossarch.csproj" Category="clr" />
 
     <ProjectToBuild Condition="'$(TargetOS)' == 'windows' or ('$(TargetOS)' == 'linux' and ('$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'arm64')) or '$(TargetOS)' == 'osx'" Include="$(CoreClrProjectRoot)tools\SuperFileCheck\SuperFileCheck.csproj" Category="clr" />
@@ -367,8 +367,8 @@
 
   <ItemGroup Condition="$(_subset.Contains('+clr.nativecorelib+'))">
     <!-- Build crossgen2 that will be used to compile System.Private.CoreLib library for CoreCLR -->
-    <ProjectToBuild Condition="'$(TargetArchitecture)' != 'x64' and '$(BuildArchitecture)' == 'x64'" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_crossarch.csproj" Category="clr" />
-    <ProjectToBuild Condition="!('$(TargetArchitecture)' != 'x64' and '$(BuildArchitecture)' == 'x64')" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
+    <ProjectToBuild Condition="('$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)') and '$(BuildArchitecture)' == 'x64'" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_crossarch.csproj" Category="clr" />
+    <ProjectToBuild Condition="!(('$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)') and '$(BuildArchitecture)' == 'x64')" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)crossgen-corelib.proj" Category="clr" />
   </ItemGroup>
 

diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
@@ -76,7 +76,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "s390x")
   set(TOOLCHAIN "s390x-linux-gnu")
 elseif(TARGET_ARCH_NAME STREQUAL "x64")
   set(CMAKE_SYSTEM_PROCESSOR x86_64)
-  if(LINUX)
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl)
+    set(TOOLCHAIN "x86_64-alpine-linux-musl")
+  elseif(LINUX)
     set(TOOLCHAIN "x86_64-linux-gnu")
     if(TIZEN)
       set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0")
@@ -279,7 +281,7 @@ endif()
 
 # Specify compile options
 
-if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
+if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
   set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
   set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
   set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})

diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets
@@ -25,7 +25,7 @@
     <CoreCLRSharedFrameworkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework'))</CoreCLRSharedFrameworkDir>
     <CoreCLRCrossgen2Dir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'crossgen2'))</CoreCLRCrossgen2Dir>
     <CoreCLRILCompilerDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'ilc-published'))</CoreCLRILCompilerDir>
-    <CoreCLRCrossILCompilerDir Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)'">$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'ilc'))</CoreCLRCrossILCompilerDir>
+    <CoreCLRCrossILCompilerDir Condition="'$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)'">$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'ilc'))</CoreCLRCrossILCompilerDir>
     <CoreCLRAotSdkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'aotsdk'))</CoreCLRAotSdkDir>
     <CoreCLRBuildIntegrationDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'build'))</CoreCLRBuildIntegrationDir>
 

diff --git a/eng/native/build-commons.sh b/eng/native/build-commons.sh
@@ -473,6 +473,7 @@ while :; do
         hostarch|-hostarch)
             if [[ -n "$2" ]]; then
                 __HostArch="$2"
+                __ExplicitHostArch=1
                 shift
             else
                 echo "ERROR: 'hostarch' requires a non-empty option argument"

diff --git a/eng/native/tryrun.cmake b/eng/native/tryrun.cmake
@@ -12,6 +12,7 @@ if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf OR
    EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl OR
    EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/ppc64le-alpine-linux-musl OR
    EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl OR
+   EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl OR
    EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl)
 
   set(ALPINE_LINUX 1)
@@ -74,7 +75,7 @@ if(DARWIN)
   else()
     message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm64 or x64 is supported for OSX cross build!")
   endif()
-elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s390x|ppc64le|x86)$" OR FREEBSD OR ILLUMOS OR TIZEN)
+elseif(TARGET_ARCH_NAME MATCHES "^(armel|arm|armv6|arm64|loongarch64|riscv64|s390x|ppc64le|x86|x64)$" OR FREEBSD OR ILLUMOS OR TIZEN)
   set_cache_value(FILE_OPS_CHECK_FERROR_OF_PREVIOUS_CALL_EXITCODE 1)
   set_cache_value(GETPWUID_R_SETS_ERRNO_EXITCODE 0)
   set_cache_value(HAS_POSIX_SEMAPHORES_EXITCODE 0)
@@ -160,6 +161,6 @@ else()
   message(FATAL_ERROR "Unsupported platform. OS: ${CMAKE_SYSTEM_NAME}, arch: ${TARGET_ARCH_NAME}")
 endif()
 
-if(TARGET_ARCH_NAME MATCHES "^(x86|s390x|armv6|loongarch64|riscv64|ppc64le)$")
+if(TARGET_ARCH_NAME MATCHES "^(x86|x64|s390x|armv6|loongarch64|riscv64|ppc64le)$")
   set_cache_value(HAVE_FUNCTIONAL_PTHREAD_ROBUST_MUTEXES_EXITCODE 0)
 endif()
diff --git a/eng/nativepgo.targets b/eng/nativepgo.targets
@@ -1,7 +1,8 @@
 <Project>
   <PropertyGroup>
     <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'windows' And ('$(TargetArchitecture)' == 'x64' Or '$(TargetArchitecture)' == 'x86' Or '$(TargetArchitecture)' == 'arm64')">true</NativeOptimizationDataSupported>
-    <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'linux' And ('$(TargetArchitecture)' == 'x64' Or '$(TargetArchitecture)' == 'arm64')">true</NativeOptimizationDataSupported>
+    <!-- Disable PGO until we get profile data collected with a new clang -->
+    <NativeOptimizationDataSupported Condition="'$(TargetOS)' == 'linux' And ('$(TargetArchitecture)' == 'x64' Or '$(TargetArchitecture)' == 'arm64')">false</NativeOptimizationDataSupported>
     <NativeOptimizationDataSupported Condition="'$(NoPgoOptimize)' == 'true'">false</NativeOptimizationDataSupported>
     <NativeOptimizationDataSupported Condition="'$(Configuration)' != 'Release'">false</NativeOptimizationDataSupported>
 

diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -75,9 +75,11 @@ jobs:
         - name: _osParameter
           value: /p:RuntimeOS=linux-bionic
 
-      # Do not rename as it clashes with MSBuild property in libraries/build-native.proj
-      - name: _crossBuildPropertyArg
-        value: /p:CrossBuild=${{ parameters.crossBuild }}
+      - name: crossArg
+        value: ''
+      - ${{ if eq(parameters.crossBuild, true) }}:
+        - name: crossArg
+          value: '-cross'
 
       - ${{ if ne(parameters.jobParameters.crossrootfsDir, '') }}:
         # This is only required for cross builds.
@@ -187,7 +189,7 @@ jobs:
         - task: CodeQL3000Init@0
           displayName: Initialize CodeQL (manually-injected)
 
-      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_crossBuildPropertyArg) $(_cxx11Parameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter)
+      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_cxx11Parameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter)
         displayName: Build product
         ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}:
           continueOnError: ${{ parameters.shouldContinueOnError }}

diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml
@@ -108,6 +108,7 @@ jobs:
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         buildConfig: ${{ parameters.buildConfig }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        crossBuild: true
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux musl arm
@@ -222,6 +223,7 @@ jobs:
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         buildConfig: ${{ parameters.buildConfig }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        crossBuild: true
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux x86
@@ -316,6 +318,68 @@ jobs:
         isNonPortableSourceBuild: false
         ${{ insert }}: ${{ parameters.jobParameters }}
 
+# GCC Linux x64 Build
+
+- ${{ if containsValue(parameters.platforms, 'gcc_linux_x64') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: linux
+      archType: x64
+      targetRid: linux-x64
+      platform: linux_x64
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container: debian-11-gcc12-amd64
+      jobParameters:
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        buildConfig: ${{ parameters.buildConfig }}
+        helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        compilerName: gcc
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Mono Linux arm64 product build
+
+- ${{ if containsValue(parameters.platforms, 'mono_linux_arm64') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: linux
+      archType: arm64
+      targetRid: linux-arm64
+      platform: linux_arm64
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container: mono_linux_arm64
+      jobParameters:
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        buildConfig: ${{ parameters.buildConfig }}
+        helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        crossBuild: true
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Mono LLVMAot test build
+
+- ${{ if containsValue(parameters.platforms, 'linux_x64_llvmaot') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: linux
+      archType: x64
+      targetRid: linux-x64
+      platform: linux_x64
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container: linux_x64_llvmaot
+      jobParameters:
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        buildConfig: ${{ parameters.buildConfig }}
+        helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
 # Linux s390x
 
 - ${{ if containsValue(parameters.platforms, 'linux_s390x') }}:

diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml
@@ -5,7 +5,7 @@ parameters:
 resources:
   containers:
     - container: linux_arm
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-20230410173854-290f679
       env:
         ROOTFS_DIR: /crossrootfs/arm
 
@@ -14,21 +14,29 @@ resources:
       env:
         ROOTFS_DIR: /crossrootfs/armv6
 
-    - container: linux_arm64
+    # Use old build images until we can avoid a clang-12 crash: https://github.com/dotnet/runtime/issues/84503
+    - container: mono_linux_arm64
       image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-arm64
       env:
         ROOTFS_DIR: /crossrootfs/arm64
 
+    - container: linux_arm64
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-20230410173854-2288685
+      env:
+        ROOTFS_DIR: /crossrootfs/arm64
+
     - container: linux_musl_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine-20230410173854-2288685
+      env:
+        ROOTFS_DIR: /crossrootfs/x64
 
     - container: linux_musl_arm
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine-20230410173854-290f679
       env:
         ROOTFS_DIR: /crossrootfs/arm
 
     - container: linux_musl_arm64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm64-alpine
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine-20230410173854-2288685
       env:
         ROOTFS_DIR: /crossrootfs/arm64
     # This container contains all required toolsets to build for Android and for Linux with bionic libc.
@@ -37,10 +45,12 @@ resources:
       image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-android
 
     - container: linux_x64
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-20230410173854-2288685
+      env:
+        ROOTFS_DIR: /crossrootfs/x64
 
     - container: linux_x86
-      image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-x86-linux
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86-20230410173854-2288685
       env:
         ROOTFS_DIR: /crossrootfs/x86
 
@@ -59,6 +69,12 @@ resources:
       env:
         ROOTFS_DIR: /crossrootfs/ppc64le
 
+    - container: debian-11-gcc12-amd64
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-gcc12-amd64
+
+    - container: linux_x64_llvmaot
+      image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7
+
     - container: browser_wasm
       image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-net8-20230322221728-80fdceb
 

diff --git a/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml
@@ -33,7 +33,7 @@ steps:
     - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) ci ${{ parameters.archType }} $(buildConfigUpper) $(priorityArg) $(runtimeFlavorArgs) ${{ parameters.testBuildArgs }} $(runtimeVariantArg) /p:LibrariesConfiguration=${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }}
       displayName: Build Tests
   - ${{ if ne(parameters.osGroup, 'windows') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) ci os ${{ parameters.osGroup }} ${{ parameters.archType }} $(buildConfigUpper) $(priorityArg) $(runtimeFlavorArgs) ${{ parameters.testBuildArgs }} $(runtimeVariantArg) /p:LibrariesConfiguration=${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }}
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(crossArg) ci os ${{ parameters.osGroup }} ${{ parameters.archType }} $(buildConfigUpper) $(priorityArg) $(runtimeFlavorArgs) ${{ parameters.testBuildArgs }} $(runtimeVariantArg) /p:LibrariesConfiguration=${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }}
       displayName: Build Tests
 
 

diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml
@@ -13,6 +13,7 @@ parameters:
   runtimeVariant: ''
   dependsOn: []
   dependOnEvaluatePaths: false
+  crossBuild: false
   pgoType: ''
 
 ### Build managed test components (native components are getting built as part
@@ -31,16 +32,13 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    crossBuild: ${{ parameters.crossBuild }}
     container: ${{ parameters.container }}
     runtimeVariant: ${{ parameters.runtimeVariant }}
     testGroup: ${{ parameters.testGroup }}
     pool: ${{ parameters.pool }}
     dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
-
-    ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '_musl'))), not(eq(parameters.osGroup, 'osx'))) }}:
-      compilerArg: '-clang9'
-    ${{ if not(and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '_musl'))), not(eq(parameters.osGroup, 'osx')))) }}:
-      compilerArg: ''
+    compilerArg: ''
 
     # Test jobs should continue on error for internal builds
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -74,17 +72,6 @@ jobs:
         value: 'libs.sfx+libs.oob+clr.iltools /p:RefOnly=true -c Release -ci'
       - name: compilerArg
         value: ''
-      - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}:
-        - name: compilerArg
-          value: '-clang9'
-        # We need to use the stable version available on Alpine Linux
-      - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '_musl')) }}:
-        - name: compilerArg
-          value: ''
-      # AppleClang has different version scheme, so we let compiler introspection pick up the available clang from PATH
-      - ${{ if eq(parameters.osGroup, 'osx') }}:
-        - name: compilerArg
-          value: ''
 
       - name: runtimeFlavorArgs
         value: ''
@@ -109,7 +96,7 @@ jobs:
         displayName: Install native dependencies
 
     # Build core/libraries dependencies of test build
-    - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(compilerArg)
+    - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(crossArg) $(compilerArg)
       displayName: Build coreclr/libs components needed by test build
 
     - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: