Use Common Matrix Generation (#38324)

* convert python to using common matrix generation present in eng/common generate-matrix yml
Azure · Nov 19, 2024 · 67aa17f · 67aa17f
1 parent cd22b6e
commit 67aa17f
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 95 deletions.
diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
@@ -244,20 +244,26 @@ jobs:
       MatrixConfigs: ${{ parameters.MatrixConfigs }}
       MatrixFilters: ${{ parameters.MatrixFilters }}
       MatrixReplace: ${{ parameters.MatrixReplace }}
-      PreGenerationSteps:
-
-        - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
-
-        - task: Powershell@2
-          inputs:
-            pwsh: true
-            filePath: eng/scripts/distribute-packages-to-matrix.ps1
-            arguments: >-
-              -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
-              -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
-          displayName: 'Distribute Packages to Matrix'
-          condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto'))
-
+      ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+        SparseCheckoutPaths:
+          - "**/setup.py"
+          - "**/pyproject.toml"
+          - "tools/"
+          - "**/ci*.yml"
+          - "**/_version.py"
+          - "**/version.py"
+          - "**/README.md"
+          - "**/CHANGELOG.md"
+          - "**/*matrix*.json"
+        EnablePRGeneration: true
+        PRMatrixSetting: "TargetingString"
+        PreGenerationSteps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.12'
+          - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+            parameters:
+              ServiceDirectory: ${{parameters.ServiceDirectory}}
       CloudConfig:
         Cloud: Public
       AdditionalParameters:

diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml
@@ -59,57 +59,9 @@ steps:
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
-  # now we need to call Save-Package-Properties so that we can filter on it
-  - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
-    - pwsh: |
-        mkdir -p $(Build.ArtifactStagingDirectory)/diff
-      displayName: Create PR Diff Folder
-
-    - pwsh: |
-        $location = Join-Path "$(Build.ArtifactStagingDirectory)" "diff"
-
-        Write-Host "./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath `"$(Build.SourcesDirectory)`" -ArtifactPath `"$location`""
-        ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location"
-      displayName: Generate PR Diff
-
-    - pwsh: |
-        Write-Host "We freshly generated the PR diff, and this is what it sees!"
-        Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName }
-        cat $(Build.ArtifactStagingDirectory)/diff/diff.json
-      displayName: Dump PR Diff
-
-    - task: Powershell@2
-      inputs:
-        filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-        arguments: >
-          -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json
-          -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-        pwsh: true
-        workingDirectory: $(Pipeline.Workspace)
-      displayName: Save package properties filtered for PR
-
-  - ${{ else }}:
-    - task: Powershell@2
-      condition: and(succeeded(), eq(variables['SetDevVersion'],'true'))
-      inputs:
-        filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-        arguments: >
-          -ServiceDirectory ${{parameters.ServiceDirectory}}
-          -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-          -AddDevVersion
-        pwsh: true
-        workingDirectory: $(Pipeline.Workspace)
-        displayName: Save package properties with dev version
-    - task: Powershell@2
-      condition: and(succeeded(), ne(variables['SetDevVersion'],'true'))
-      inputs:
-        filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-        arguments: >
-          -ServiceDirectory ${{parameters.ServiceDirectory}}
-          -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-        pwsh: true
-        workingDirectory: $(Pipeline.Workspace)
-        displayName: Save package properties for service
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
   - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
     parameters:
@@ -136,18 +88,6 @@ steps:
     displayName: 'Prep Environment'
     condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
 
-  - task: Powershell@2
-    inputs:
-      filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-      arguments: >
-        -ServiceDirectory ${{parameters.ServiceDirectory}}
-        -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-        -AddDevVersion
-      pwsh: true
-      workingDirectory: $(Pipeline.Workspace)
-    displayName: Update package properties with dev version
-    condition: and(succeeded(), eq(variables['SetDevVersion'],'true'), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
-
   - script: |
       sudo apt-get update
       sudo apt-get install -y qemu-user-static

diff --git a/sdk/attestation/ci.yml b/sdk/attestation/ci.yml
@@ -28,12 +28,6 @@ extends:
   parameters:
     ServiceDirectory: attestation
     TestProxy: true
-    # Override the base matrix due to https://github.com/Azure/azure-sdk-for-python/issues/17837
-    MatrixConfigs:
-      - Name: attestation_ci_matrix
-        Path: eng/pipelines/templates/stages/platform-matrix.json
-        Selection: sparse
-        GenerateVMJobs: true
     Artifacts:
     - name: azure-mgmt-attestation
       safeName: azuremgmtattestation

diff --git a/sdk/evaluation/ci.yml b/sdk/evaluation/ci.yml
@@ -30,11 +30,8 @@ extends:
     TestProxy: true
     # This custom matrix config should be dropped once:
     #  * Once azure-ai-ml supports 3.13 (currently crashes due to type annotation)
-    MatrixConfigs:
-      - Name: evaluation_ci_matrix
-        Path: eng/pipelines/templates/stages/platform-matrix-no-313.json
-        Selection: sparse
-        GenerateVMJobs: true
+    MatrixFilters:
+      - PythonVersion=^(?!3\.13)
     Artifacts:
     - name: azure-ai-evaluation
       safeName: azureaievaluation
diff --git a/sdk/identity/azure-identity/LICENSE b/sdk/identity/azure-identity/LICENSE
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.
diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml
@@ -31,12 +31,6 @@ extends:
   parameters:
     ServiceDirectory: keyvault
     TestProxy: true
-    # Override the base matrix due to https://github.com/Azure/azure-sdk-for-python/issues/17837
-    MatrixConfigs:
-      - Name: keyvault_ci_matrix
-        Path: eng/pipelines/templates/stages/platform-matrix.json
-        Selection: sparse
-        GenerateVMJobs: true
     Artifacts:
     - name: azure-keyvault-administration
       safename: azurekeyvaultadministration