More pr pipeline prep (#43822)

* More prep for the PR pipeline

* fix daily-dev-build-variable.yml condition to only skip the PR pipline

* fiddle with the daily-dev-build-variable.yml condition

eng/pipelines/scripts/generate-project-list.ps1

@@ -19,15 +19,38 @@ if ($env:ADDITIONALMODULESJSON -and $env:ADDITIONALMODULESJSON -notlike '*Additi
   }
 }
 
+# If the project list is empty this is because the Artifacts and AdditionalModules are both empty
+# which means this is running as part of the pullrequest pipeline and the project list needs to
+# be figured out from the packageInfo files.
+if ($projectList.Length -eq 0 -and $ENV:PACKAGEINFODIR) {
+  [array]$packageInfoFiles = Get-ChildItem -Path $ENV:PACKAGEINFODIR "*.json"
+  foreach($packageInfoFile in $packageInfoFiles) {
+    $packageInfoJson = Get-Content $packageInfoFile -Raw
+    $packageInfo = ConvertFrom-Json $packageInfoJson
+    $fullArtifactName = "$($packageInfo.Group):$($packageInfo.ArtifactName)"
+    $projectList += $fullArtifactName
+    $artifactsList += $fullArtifactName
+    # The AdditionalValidationPackages are stored as <group>:<artifact>
+    foreach($additionalModule in $packageInfo.AdditionalValidationPackages)
+    {
+      $projectList += $additionalModule
+      $additionalModulesList += $additionalModule
+    }
+  }
+}
+
+$projectList = $projectList | Select-Object -Unique
 $projects = $projectList -join ','
 if (!$projects) {
     throw "parameters.Artifacts cannot be empty"
 }
 
+$artifactsList = $artifactsList | Select-Object -Unique
 $artifactsString = $artifactsList -join ','
 Write-Host "ArtifactsList = $artifactsString"
 Write-Host "##vso[task.setvariable variable=ArtifactsList;]$artifactsString"
 
+$additionalModulesList = $additionalModulesList | Select-Object -Unique
 $additionalModulesString = $additionalModulesList -join ','
 Write-Host "AdditionalModulesList = $additionalModulesString"
 Write-Host "##vso[task.setvariable variable=AdditionalModulesList;]$additionalModulesString"

eng/pipelines/templates/jobs/ci.yml

@@ -67,6 +67,8 @@ jobs:
 
     variables:
       ArtifactName: 'packages'
+      # The ServiceDirectory is an identifier. The pullrequest's ServiceDirectory of
+      # 'auto' shouldn't cause any problems
       Codeql.Enabled: true
       Codeql.BuildIdentifier: ${{ parameters.ServiceDirectory }}
       Codeql.SkipTaskAutoInjection: false
@@ -86,9 +88,18 @@ jobs:
             Paths:
               - '**/*.xml'
               - '**/*.md'
+              - '**/ci*.yml' # necessary for save-package-properties.yml
               - '!sdk/**/test-recordings'
               - '!sdk/**/session-records'
 
+      # This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'.
+      # For the PullRequest pipeline, the packageInfo files generated from the diff end up
+      # being the source of truth for the artifacts list.
+      - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
+        - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+          parameters:
+            ServiceDirectory: ${{parameters.ServiceDirectory}}
+
       - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
         parameters:
           JobType: 'Build'
@@ -130,9 +141,13 @@ jobs:
           ServiceDirectory: "template"
           TestPipeline: ${{ parameters.TestPipeline }}
 
-      - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml
-        parameters:
-          ServiceDirectory: ${{parameters.ServiceDirectory}}
+      # The daily dev build variable isn't used for the PullRequests pipeline but is currently used
+      # to save off the packageInfo files for all ci pipeline. This needs to be skipped for the
+      # PR pipeline, whose ServiceDirectory is 'auto' since the packageInfo files will be generated above.
+      - ${{if ne(parameters.ServiceDirectory, 'auto')}}:
+        - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml
+          parameters:
+            ServiceDirectory: ${{parameters.ServiceDirectory}}
 
       - task: PowerShell@2
         displayName: Setup Dev versioning
@@ -234,8 +249,6 @@ jobs:
           Artifacts: ${{parameters.ReleaseArtifacts}}
 
       - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml
-        parameters:
-          Artifacts: ${{parameters.Artifacts}}
 
       - template: /eng/common/pipelines/templates/steps/validate-all-packages.yml
         parameters:

eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml

@@ -29,6 +29,7 @@ steps:
   env:
     ARTIFACTSJSON: $(ArtifactsJson)
     ADDITIONALMODULESJSON: $(AdditionalModulesJson)
+    PACKAGEINFODIR: $(Build.ArtifactStagingDirectory)/PackageInfo
 
 # The Cache task, when restoring, can have a miss and still pass but, if nothing is built, the
 # Post-Job Cache will fail because there's no .m2/repository