Remove various single-scenario YAML parameters in our build scripts and shift implementation responsibility to those scenarios

eng/pipelines/common/global-build-job.yml

@@ -25,20 +25,14 @@ parameters:
  useContinueOnErrorDuringBuild: false
  shouldContinueOnError: false
  isOfficialBuild: false
- isSourceBuild: false
- isNonPortableSourceBuild: false
  runtimeFlavor: 'coreclr'
  runtimeVariant: ''
  helixQueues: ''
  enablePublishTestResults: false
  testResultsFormat: ''
  postBuildSteps: []
  extraVariablesTemplates: []
- isManualCodeQLBuild: false
  preBuildSteps: []
- enableRichCodeNavigation: false
- richCodeNavigationLanguage: 'csharp'
- disableComponentGovernance: ''
  templatePath: 'templates'
 
 jobs:
@@ -57,9 +51,6 @@ jobs:
  enablePublishTestResults: ${{ parameters.enablePublishTestResults }}
  testResultsFormat: ${{ parameters.testResultsFormat }}
 
- enableRichCodeNavigation: ${{ parameters.enableRichCodeNavigation }}
- richCodeNavigationLanguage: ${{ parameters.richCodeNavigationLanguage }}
-
  artifacts:
  publish:
  logs:
@@ -154,10 +145,7 @@ jobs:
 
  - checkout: self
  clean: true
- # If running in source build mode, a git stash will be used for the inner clone. Avoid setting a fetch depth,
- # as a stash of a shallow cloned repo is not currently supported.
- ${{ if ne(parameters.isSourceBuild, true) }}:
- fetchDepth: $(checkoutFetchDepth)
+ fetchDepth: $(checkoutFetchDepth)
 
  - ${{ if and(eq(parameters.isOfficialBuild, true), notin(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator')) }}:
  - template: /eng/pipelines/common/restore-internal-tools.yml
@@ -187,16 +175,6 @@ jobs:
  artifact: Mono_Offsets_${{monoCrossAOTTargetOS}}
  path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'
 
- - ${{ if eq(parameters.isSourceBuild, true) }}:
- - template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml
- parameters:
- platform:
- baseOS: ${{ parameters.baseOS }}
- buildScript: $(Build.SourcesDirectory)$(dir)build$(scriptExt)
- nonPortable: ${{ parameters.isNonPortableSourceBuild }}
- targetRID: ${{ parameters.targetRid }}
- name: ${{ parameters.platform }}
-
  - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}:
  - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }}
  displayName: Install Build Dependencies
@@ -228,7 +206,6 @@ jobs:
  targetRid: ${{ parameters.targetRid }}
  nameSuffix: ${{ parameters.nameSuffix }}
  platform: ${{ parameters.platform }}
- pgoType: ${{ parameters.pgoType }}
  shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
  ${{ if ne(preBuildStep.forwardedParameters, '') }}:
  ${{ each parameter in preBuildStep.forwardedParameters }}:
@@ -239,21 +216,11 @@ jobs:
  - ${{ preBuildStep }}
 
  # Build
- - ${{ if eq(parameters.isSourceBuild, false) }}:
- - ${{ if eq(parameters.isManualCodeQLBuild, true) }}:
- - task: CodeQL3000Init@0
- displayName: Initialize CodeQL (manually-injected)
-
- - template: /eng/pipelines/common/templates/global-build-step.yml
- parameters:
- buildArgs: ${{ parameters.buildArgs }}
- useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }}
- shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-
- - ${{ if eq(parameters.isManualCodeQLBuild, true) }}:
- - task: CodeQL3000Finalize@0
- displayName: Finalize CodeQL (manually-injected)
- #endif isSourceBuild
+ - template: /eng/pipelines/common/templates/global-build-step.yml
+ parameters:
+ buildArgs: ${{ parameters.buildArgs }}
+ useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }}
+ shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
 
  - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos', 'android') }}:
  - script: |

eng/pipelines/common/platform-matrix.yml

@@ -294,73 +294,6 @@ jobs:
  helixQueueGroup: ${{ parameters.helixQueueGroup }}
  ${{ insert }}: ${{ parameters.jobParameters }}
 
-# Centos 8 Stream x64 Source Build
-
-- ${{ if containsValue(parameters.platforms, 'SourceBuild_centos8_x64') }}:
- - template: xplat-setup.yml
- parameters:
- jobTemplate: ${{ parameters.jobTemplate }}
- helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
- variables: ${{ parameters.variables }}
- osGroup: linux
- archType: x64
- targetRid: centos.8-x64
- platform: centos8_linux_x64
- shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
- container: SourceBuild_centos_x64
- jobParameters:
- runtimeFlavor: ${{ parameters.runtimeFlavor }}
- buildConfig: ${{ parameters.buildConfig }}
- helixQueueGroup: ${{ parameters.helixQueueGroup }}
- isSourceBuild: true
- isNonPortableSourceBuild: true
- ${{ insert }}: ${{ parameters.jobParameters }}
-
-# Non-existent RID Source Build
-
-- ${{ if containsValue(parameters.platforms, 'SourceBuild_banana24_x64') }}:
- - template: xplat-setup.yml
- parameters:
- jobTemplate: ${{ parameters.jobTemplate }}
- helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
- variables: ${{ parameters.variables }}
- osGroup: linux
- archType: x64
- targetRid: banana.24-x64
- platform: banana24_linux_x64
- shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
- container: SourceBuild_centos_x64 # Run the unknown-rid build on a platform with a known RID so our RID graph tooling can automatically add it to the RID graph.
- jobParameters:
- runtimeFlavor: ${{ parameters.runtimeFlavor }}
- baseOS: linux
- buildConfig: ${{ parameters.buildConfig }}
- helixQueueGroup: ${{ parameters.helixQueueGroup }}
- isSourceBuild: true
- isNonPortableSourceBuild: true
- ${{ insert }}: ${{ parameters.jobParameters }}
-
-# Portable Linux x64 Source Build
-
-- ${{ if containsValue(parameters.platforms, 'SourceBuild_linux_x64') }}:
- - template: xplat-setup.yml
- parameters:
- jobTemplate: ${{ parameters.jobTemplate }}
- helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
- variables: ${{ parameters.variables }}
- osGroup: linux
- archType: x64
- targetRid: linux-x64
- platform: linux_x64
- shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
- container: SourceBuild_linux_x64
- jobParameters:
- runtimeFlavor: ${{ parameters.runtimeFlavor }}
- buildConfig: ${{ parameters.buildConfig }}
- helixQueueGroup: ${{ parameters.helixQueueGroup }}
- isSourceBuild: true
- isNonPortableSourceBuild: false
- ${{ insert }}: ${{ parameters.jobParameters }}
-
 # GCC Linux x64 Build
 
 - ${{ if containsValue(parameters.platforms, 'gcc_linux_x64') }}:

eng/pipelines/global-build.yml

@@ -163,16 +163,11 @@ extends:
  eq(variables['isRollingBuild'], true))
 
  #
- # SourceBuild Build
+ # Build Portable SourceBuild
  #
- - template: /eng/pipelines/common/platform-matrix.yml
+ - template: /eng/common/templates/jobs/source-build.yml
  parameters:
- jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: Release
  platforms:
- - SourceBuild_linux_x64
- jobParameters:
- nameSuffix: PortableSourceBuild
- timeoutInMinutes: 95
- condition:
- eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true)
+ - name: Linux_x64
+ targetRid: linux-x64
+ container: SourceBuild_linux_x64

eng/pipelines/runtime-codeql.yml

@@ -47,7 +47,12 @@ extends:
  nameSuffix: AllSubsets_CoreCLR
  isOfficialBuild: ${{ variables.isOfficialBuild }}
  timeoutInMinutes: 360
- isManualCodeQLBuild: true
+ preBuildSteps:
+ - task: CodeQL3000Init@0
+ displayName: Initialize CodeQL (manually-injected)
+ postBuildSteps:
+ - task: CodeQL3000Finalize@0
+ displayName: Finalize CodeQL (manually-injected)
 
  #
  # Build Mono runtime packs
@@ -65,4 +70,9 @@ extends:
  nameSuffix: AllSubsets_Mono
  isOfficialBuild: ${{ variables.isOfficialBuild }}
  timeoutInMinutes: 360
- isManualCodeQLBuild: true
+ preBuildSteps:
+ - task: CodeQL3000Init@0
+ displayName: Initialize CodeQL (manually-injected)
+ postBuildSteps:
+ - task: CodeQL3000Finalize@0
+ displayName: Finalize CodeQL (manually-injected)

eng/pipelines/runtime-official.yml

@@ -560,22 +560,12 @@ extends:
  #
  # Build SourceBuild packages
  #
- - template: /eng/pipelines/common/platform-matrix.yml
+ - template: /eng/common/templates-official/jobs/source-build.yml
  parameters:
- jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: Release
- helixQueueGroup: ci
  platforms:
- - SourceBuild_linux_x64
- jobParameters:
- templatePath: 'templates-official'
- nameSuffix: PortableSourceBuild
- isOfficialBuild: ${{ variables.isOfficialBuild }}
- postBuildSteps:
- - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
- parameters:
- name: SourceBuildPackages
- timeoutInMinutes: 95
+ - name: Linux_x64
+ targetRid: linux-x64
+ container: SourceBuild_linux_x64
 
  #
  # Build PGO Instrumented CoreCLR Release

eng/pipelines/runtime.yml

@@ -1878,36 +1878,29 @@ extends:
  eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true),
  eq(variables['isRollingBuild'], true))
 
- #
- # Sourcebuild legs
- # We have 3 important legs for source-build:
- # - Centos.8 (ensures that known non-portable RID is working)
- # - Linux-x64 portable (used for dependency flow and downstream PR verification)
- # - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about.
- #
- # Running all of these everywhere is wasteful. Run Banana.24 and CentOS.8 in rolling CI,
- # Run Linux-x64 in PR.
-
- - template: /eng/pipelines/common/platform-matrix.yml
- parameters:
- jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: Release
- helixQueueGroup: pr
- platforms:
- - SourceBuild_centos8_x64
- jobParameters:
- nameSuffix: centos8SourceBuild
- timeoutInMinutes: 95
- condition: eq(variables['isRollingBuild'], true)
-
- - template: /eng/pipelines/common/platform-matrix.yml
- parameters:
- jobTemplate: /eng/pipelines/common/global-build-job.yml
- buildConfig: Release
- helixQueueGroup: pr
- platforms:
- - SourceBuild_banana24_x64
- jobParameters:
- nameSuffix: banana24SourceBuild
- timeoutInMinutes: 95
- condition: eq(variables['isRollingBuild'], true)
+ - stage: SourceBuild
+ displayName: Source Build Validation
+ dependsOn: []
+ condition: eq(variables['isRollingBuild'], true)
+ jobs:
+ #
+ # Sourcebuild legs
+ # We have 3 important legs for source-build:
+ # - Centos.8 (ensures that known non-portable RID is working)
+ # - Linux-x64 portable (used for dependency flow and downstream PR verification)
+ # - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about.
+ #
+ # Running all of these everywhere is wasteful. Run Banana.24 and CentOS.8 in rolling CI,
+ # Run Linux-x64 in PR.
+ - template: /eng/common/templates/jobs/source-build.yml
+ parameters:
+ platforms:
+ - name: CentOS8
+ targetRid: centos.8-x64
+ nonPortable: true
+ container: SourceBuild_centos_x64
+ - name: NonexistentRID
+ baseOS: linux
+ targetRid: banana.24-x64
+ nonPortable: true
+ container: SourceBuild_centos_x64

src/installer/prepare-artifacts.proj

@@ -58,8 +58,8 @@
  <ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
  </ItemGroup>
 
- <!-- 
- Set metadata for assets that are not marked as NonShipping. 
+ <!--
+ Set metadata for assets that are not marked as NonShipping.
  This is used to determine if the asset should be shipped as part of .NET release.
  -->
  <ItemDefinitionGroup>
@@ -145,10 +145,6 @@
  <RelativeBlobPath>$(InstallersRelativePath)workloads/%(Filename)%(Extension)</RelativeBlobPath>
  <PublishFlatContainer>true</PublishFlatContainer>
  </ItemsToPush>
-
- <!-- Source build intermediated packages will be pushed and signed by the sourcebuild leg. -->
- <ItemsToPush Remove="@(ItemsToPush)" Condition="$([System.String]::new('%(Identity)').Contains('Microsoft.SourceBuild.Intermediate'))" />
- <ItemsToSignPostBuild Remove="@(ItemsToSignPostBuild)" Condition="$([System.String]::new('%(Identity)').Contains('Microsoft.SourceBuild.Intermediate'))" />
  </ItemGroup>
 
  <!-- Push items to AzDO as build artifacts, generating the asset manifest as a side effect. -->