From 8e5e23b1ba224850de11d26d1185562fe34338ba Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 16 Jan 2025 12:07:55 -0800 Subject: [PATCH 1/3] More prep for the PR pipeline --- .../scripts/generate-project-list.ps1 | 23 +++++++++++++++++++ eng/pipelines/templates/jobs/ci.yml | 21 +++++++++++++---- ...roject-list-and-cache-maven-repository.yml | 1 + 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/scripts/generate-project-list.ps1 b/eng/pipelines/scripts/generate-project-list.ps1 index 72dded99fd8cc..836763022c2f4 100644 --- a/eng/pipelines/scripts/generate-project-list.ps1 +++ b/eng/pipelines/scripts/generate-project-list.ps1 @@ -19,15 +19,38 @@ if ($env:ADDITIONALMODULESJSON -and $env:ADDITIONALMODULESJSON -notlike '*Additi } } +# If the project list is empty this is because the Artifacts and AdditionalModules are both empty +# which means this is running as part of the pullrequest pipeline and the project list needs to +# be figured out from the packageInfo files. +if ($projectList.Length -eq 0 -and $ENV:PACKAGEINFODIR) { + [array]$packageInfoFiles = Get-ChildItem -Path $ENV:PACKAGEINFODIR "*.json" + foreach($packageInfoFile in $packageInfoFiles) { + $packageInfoJson = Get-Content $packageInfoFile -Raw + $packageInfo = ConvertFrom-Json $packageInfoJson + $fullArtifactName = "$($packageInfo.Group):$($packageInfo.ArtifactName)" + $projectList += $fullArtifactName + $artifactsList += $fullArtifactName + # The AdditionalValidationPackages are stored as : + foreach($additionalModule in $packageInfo.AdditionalValidationPackages) + { + $projectList += $additionalModule + $additionalModulesList += $additionalModule + } + } +} + +$projectList = $projectList | Select-Object -Unique $projects = $projectList -join ',' if (!$projects) { throw "parameters.Artifacts cannot be empty" } +$artifactsList = $artifactsList | Select-Object -Unique $artifactsString = $artifactsList -join ',' Write-Host "ArtifactsList = $artifactsString" Write-Host "##vso[task.setvariable variable=ArtifactsList;]$artifactsString" +$additionalModulesList = $additionalModulesList | Select-Object -Unique $additionalModulesString = $additionalModulesList -join ',' Write-Host "AdditionalModulesList = $additionalModulesString" Write-Host "##vso[task.setvariable variable=AdditionalModulesList;]$additionalModulesString" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index ab9251e275736..4887abceed3df 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -67,6 +67,8 @@ jobs: variables: ArtifactName: 'packages' + # The ServiceDirectory is an identifier. The pullrequest's ServiceDirectory of + # 'auto' shouldn't cause any problems Codeql.Enabled: true Codeql.BuildIdentifier: ${{ parameters.ServiceDirectory }} Codeql.SkipTaskAutoInjection: false @@ -86,9 +88,18 @@ jobs: Paths: - '**/*.xml' - '**/*.md' + - '**/ci*.yml' # necessary for save-package-properties.yml - '!sdk/**/test-recordings' - '!sdk/**/session-records' + # This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'. + # For the PullRequest pipeline, the packageInfo files generated from the diff end up + # being the source of truth for the artifacts list. + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}: + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} + - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: JobType: 'Build' @@ -130,9 +141,11 @@ jobs: ServiceDirectory: "template" TestPipeline: ${{ parameters.TestPipeline }} - - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml - parameters: - ServiceDirectory: ${{parameters.ServiceDirectory}} + # The daily dev build variable isn't used for PullRequests. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} - task: PowerShell@2 displayName: Setup Dev versioning @@ -234,8 +247,6 @@ jobs: Artifacts: ${{parameters.ReleaseArtifacts}} - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml - parameters: - Artifacts: ${{parameters.Artifacts}} - template: /eng/common/pipelines/templates/steps/validate-all-packages.yml parameters: diff --git a/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml b/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml index c9eaf3ca6c87d..7eacf72434809 100644 --- a/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml +++ b/eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml @@ -29,6 +29,7 @@ steps: env: ARTIFACTSJSON: $(ArtifactsJson) ADDITIONALMODULESJSON: $(AdditionalModulesJson) + PACKAGEINFODIR: $(Build.ArtifactStagingDirectory)/PackageInfo # The Cache task, when restoring, can have a miss and still pass but, if nothing is built, the # Post-Job Cache will fail because there's no .m2/repository From 21de0e720e758671650417e91efbdeaf8cacd134 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 16 Jan 2025 12:35:22 -0800 Subject: [PATCH 2/3] fix daily-dev-build-variable.yml condition to only skip the PR pipline --- eng/pipelines/templates/jobs/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 4887abceed3df..1a1de4a681447 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -141,8 +141,10 @@ jobs: ServiceDirectory: "template" TestPipeline: ${{ parameters.TestPipeline }} - # The daily dev build variable isn't used for PullRequests. - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + # The daily dev build variable isn't used for the PullRequests pipeline but is currently used + # to save off the packageInfo files for all ci pipeline. This needs to be skipped for the + # PR pipeline since the packageInfo files will be generated above. + - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), ne(parameters.ServiceDirectory, 'auto')) }}: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} From aa6f3c1ce394c0169284b02132b699797404fc10 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 16 Jan 2025 13:14:01 -0800 Subject: [PATCH 3/3] fiddle with the daily-dev-build-variable.yml condition --- eng/pipelines/templates/jobs/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 1a1de4a681447..94153299f975a 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -143,8 +143,8 @@ jobs: # The daily dev build variable isn't used for the PullRequests pipeline but is currently used # to save off the packageInfo files for all ci pipeline. This needs to be skipped for the - # PR pipeline since the packageInfo files will be generated above. - - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), ne(parameters.ServiceDirectory, 'auto')) }}: + # PR pipeline, whose ServiceDirectory is 'auto' since the packageInfo files will be generated above. + - ${{if ne(parameters.ServiceDirectory, 'auto')}}: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}}