-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Sync eng/common directory with azure-sdk-tools for PR 7758 (#42381)
* Add 1es changes for job/matrix generation and publish * Use more flexible pool filter for prev/next pool matches * consolidate displayname definition * use linux pool variables for generate matrix job * Fix publish artifact * Use single publish task for publish artifact --------- Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>
- Loading branch information
Showing
2 changed files
with
170 additions
and
0 deletions.
There are no files selected for viewing
140 changes: 140 additions & 0 deletions
140
eng/common/pipelines/templates/jobs/generate-job-matrix.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,140 @@ | ||
parameters: | ||
- name: AdditionalParameters | ||
type: object | ||
- name: DependsOn | ||
type: object | ||
default: null | ||
- name: CloudConfig | ||
type: object | ||
default: {} | ||
- name: MatrixConfigs | ||
type: object | ||
default: [] | ||
- name: MatrixFilters | ||
type: object | ||
default: [] | ||
- name: MatrixReplace | ||
type: object | ||
default: {} | ||
- name: JobTemplatePath | ||
type: string | ||
# Set this to false to do a full checkout for private repositories with the azure pipelines service connection | ||
- name: SparseCheckout | ||
type: boolean | ||
default: true | ||
- name: SparseCheckoutPaths | ||
type: object | ||
default: [] | ||
- name: Pool | ||
type: string | ||
default: $(LINUXPOOL) | ||
- name: OsVmImage | ||
type: string | ||
default: $(LINUXVMIMAGE) | ||
- name: Os | ||
type: string | ||
default: linux | ||
# This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE. | ||
# When that occurs, provide a name other than the default value. | ||
- name: GenerateJobName | ||
type: string | ||
default: 'generate_job_matrix' | ||
- name: PreGenerationSteps | ||
type: stepList | ||
default: [] | ||
# Mappings to OS name required at template compile time by 1es pipeline templates | ||
- name: Pools | ||
type: object | ||
default: | ||
- name: Linux | ||
filter: .*Linux.*Pool$ | ||
os: linux | ||
- name: Windows | ||
filter: .*Windows.*Pool$ | ||
os: windows | ||
- name: Mac | ||
filter: .*MacPool$ | ||
os: macOS | ||
|
||
jobs: | ||
- job: ${{ parameters.GenerateJobName }} | ||
variables: | ||
- template: /eng/pipelines/templates/variables/image.yml | ||
- name: skipComponentGovernanceDetection | ||
value: true | ||
- name: displayNameFilter | ||
value: $[ coalesce(variables.jobMatrixFilter, '.*') ] | ||
pool: | ||
name: ${{ parameters.Pool }} | ||
vmImage: ${{ parameters.OsVmImage }} | ||
os: ${{ parameters.Os }} | ||
${{ if parameters.DependsOn }}: | ||
dependsOn: ${{ parameters.DependsOn }} | ||
steps: | ||
# Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories | ||
# as we require the github service connection to be loaded. | ||
- ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}: | ||
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml | ||
parameters: | ||
${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}: | ||
Paths: ${{ parameters.SparseCheckoutPaths }} | ||
${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}: | ||
Paths: | ||
- "sdk/${{ parameters.AdditionalParameters.ServiceDirectory }}" | ||
|
||
- ${{ parameters.PreGenerationSteps }} | ||
|
||
- ${{ each config in parameters.MatrixConfigs }}: | ||
- ${{ each pool in parameters.Pools }}: | ||
- ${{ if eq(config.GenerateVMJobs, 'true') }}: | ||
- task: Powershell@2 | ||
inputs: | ||
pwsh: true | ||
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 | ||
arguments: > | ||
-ConfigPath ${{ config.Path }} | ||
-Selection ${{ config.Selection }} | ||
-DisplayNameFilter '$(displayNameFilter)' | ||
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' | ||
-Replace '${{ join(''',''', parameters.MatrixReplace) }}' | ||
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' | ||
displayName: Create ${{ pool.name }} Matrix ${{ config.Name }} | ||
name: vm_job_matrix_${{ config.Name }}_${{ pool.name }} | ||
|
||
- ${{ if eq(config.GenerateContainerJobs, 'true') }}: | ||
- task: Powershell@2 | ||
inputs: | ||
pwsh: true | ||
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 | ||
arguments: > | ||
-ConfigPath ${{ config.Path }} | ||
-Selection ${{ config.Selection }} | ||
-DisplayNameFilter '$(displayNameFilter)' | ||
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' | ||
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' | ||
displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }} | ||
name: container_job_matrix_${{ config.Name }}_${{ pool.name }} | ||
|
||
- ${{ each config in parameters.MatrixConfigs }}: | ||
- ${{ each pool in parameters.Pools }}: | ||
- ${{ if eq(config.GenerateVMJobs, 'true') }}: | ||
- template: ${{ parameters.JobTemplatePath }} | ||
parameters: | ||
UsePlatformContainer: false | ||
OSName: ${{ pool.os }} | ||
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] | ||
DependsOn: ${{ parameters.GenerateJobName }} | ||
CloudConfig: ${{ parameters.CloudConfig }} | ||
${{ each param in parameters.AdditionalParameters }}: | ||
${{ param.key }}: ${{ param.value }} | ||
|
||
- ${{ if eq(config.GenerateContainerJobs, 'true') }}: | ||
- template: ${{ parameters.JobTemplatePath }} | ||
parameters: | ||
UsePlatformContainer: true | ||
OSName: ${{ pool.os }} | ||
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] | ||
DependsOn: ${{ parameters.GenerateJobName }} | ||
CloudConfig: ${{ parameters.CloudConfig }} | ||
${{ each param in parameters.AdditionalParameters }}: | ||
${{ param.key }}: ${{ param.value }} |
30 changes: 30 additions & 0 deletions
30
eng/common/pipelines/templates/steps/publish-1es-artifact.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
# This step is used to prevent duplication of artifact publishes when there is an issue that would prevent the overall success of the job. | ||
# Ensuring that we only publish when successful (and two a differently named artifact otherwise) will allow easy retry on a build pipeline | ||
# without running into the "cannot override artifact" failure when we finally do get a passing run. | ||
|
||
# ArtifactName - The name of the artifact in the "successful" case. | ||
# ArtifactPath - The path we will be publishing. | ||
# CustomCondition - Used if there is additional logic necessary to prevent attempt of publish. | ||
|
||
parameters: | ||
ArtifactName: '' | ||
ArtifactPath: '' | ||
CustomCondition: true | ||
|
||
steps: | ||
- pwsh: | | ||
Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}" | ||
condition: and(succeeded(), ${{ parameters.CustomCondition }}) | ||
displayName: Set Artifact Name | ||
- pwsh: | | ||
Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)" | ||
condition: and(failed(), ${{ parameters.CustomCondition }}) | ||
displayName: Set Failed Artifact Name | ||
- task: 1ES.PublishPipelineArtifact@1 | ||
condition: and(succeeded(), ${{ parameters.CustomCondition }}) | ||
displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts' | ||
inputs: | ||
artifact: '$(PublishArtifactName)' | ||
path: '${{ parameters.ArtifactPath }}' |