Migrate java pipelines to 1es pipeline templates (#39243)

* Migrate java pipelines to 1es pipeline templates Co-authored-by: Scott Beddall (from Dev Box) <scbedd@microsoft.com> Co-authored-by: Alan Zimmer <48699787+alzimmermsft@users.noreply.github.com> * PR, main fixes * Disable spotbugs in 1es sdl * Set storage pool replace * Fix custom live test extends (cosmos, spring, communicatiion) * Fix linux next image value * Set sdl config on public * Add cspell ignore for matrix test config files --------- Co-authored-by: Scott Beddall (from Dev Box) <scbedd@microsoft.com> Co-authored-by: Alan Zimmer <48699787+alzimmermsft@users.noreply.github.com> (cherry picked from commit 9e789e6)
Azure · Jul 15, 2024 · 1e0757b · 1e0757b
1 parent 74491a6
commit 1e0757b
Show file tree

Hide file tree

Showing 92 changed files with 2,622 additions and 1,852 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 # Default Assets restore directory
 .assets
+test-proxy.log
 
 *.class
 
@@ -118,4 +119,4 @@ stress-test-resources.json
 stress-test-addons*
 
 # Temp typespec files
-TempTypeSpecFiles/
+TempTypeSpecFiles/
diff --git a/.vscode/cspell.json b/.vscode/cspell.json
@@ -16,6 +16,7 @@
     "**/sdk/**/target/**",
     "**/session-records/**",
     "**/sdk/**/**/assets.json",
+    "**/sdk/**/*-matrix.json",
     "**/chromedriver",
     ".gitignore",
     "*.jar",

diff --git a/eng/CredScanSuppression.json b/eng/CredScanSuppression.json
@@ -35,9 +35,16 @@
             ],
             "_justification": "Javadoc in azure resourcemanagerhybrid"
         },
+        {
+            "file": [
+                "test-proxy.log",
+            ],
+            "_justification": "Transient test file that is locked by test-proxy and should not be scanned"
+        },
         {
             "file": [
                 "eng/common/testproxy/dotnet-devcert.pfx",
+                "sdk/core/azure-core-test/target/classes/keystore.jks",
                 "sdk/cosmos/azure-cosmos-tests/src/test/resources/server.jks",
                 "sdk/cosmos/azure-cosmos-tests/src/test/resources/client.jks",
                 "sdk/identity/azure-identity/src/test/resources/cert-chain.pem",
@@ -64,6 +71,9 @@
             "file": [
                 "sdk/communication/azure-communication-common-perf/src/main/java/com.azure.communication.common.perf/FakeCredentialInTest.java",
                 "sdk/keyvault/azure-security-keyvault-certificates/src/test/java/com/azure/security/keyvault/certificates/FakeCredentialInTest.java",
+                "sdk/keyvault/microsoft-azure-keyvault/src/test/resources/session-records/createCertificatePemForCertificateOperationsTest.json",
+                "sdk/keyvault/microsoft-azure-keyvault/src/test/resources/session-records/createSelfSignedCertificatePemForCertificateOperationsTest.json",
+                "sdk/resourcemanagerhybrid/azure-resourcemanager/src/test/resources/session-records/AzureResourceManagerTests.testContainerRegistry.json",
                 "sdk/spring/spring-cloud-azure-autoconfigure/src/test/java/com/azure/spring/cloud/autoconfigure/FakeCredentialInTest.java",
                 "sdk/spring/spring-cloud-azure-service/src/test/java/com/azure/spring/cloud/service/implementation/storage/FakeCredentialInTest.java",
                 "sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/FakeCredentialInTest.java",

diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml
@@ -104,10 +104,11 @@ stages:
           displayName: 'Copy BOM report to artifact staging'
           condition: succeededOrFailed()
 
-        - publish: $(Build.ArtifactStagingDirectory)
-          condition: succeededOrFailed()
-          displayName: 'Publish Report Artifacts'
-          artifact: reports
+        - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+          parameters:
+            DisplayName: 'Publish Report Artifacts'
+            ArtifactPath: '$(Build.ArtifactStagingDirectory)'
+            ArtifactName: 'reports'
 
         - template: /eng/pipelines/templates/steps/upload-repository-on-failure.yml
 
@@ -153,8 +154,9 @@ stages:
         timeoutInMinutes: 120
 
         pool:
-          name: azsdk-pool-mms-win-2022-general
-          vmImage: windows-2022
+          name: $(WINDOWSPOOL)
+          vmImage: $(WINDOWSVMIMAGE)
+          os: windows
 
         steps:
           - template: /eng/common/pipelines/templates/steps/policheck.yml

diff --git a/eng/pipelines/bompreparation.yml b/eng/pipelines/bompreparation.yml
@@ -19,6 +19,8 @@ stages:
             Copy-Item eng/scripts/bompom.html $(Build.ArtifactStagingDirectory)/staging
           displayName: 'Analyze and generate patches'
 
-        - publish: $(Build.ArtifactStagingDirectory)
-          displayName: 'Publish Report Artifacts'
-          artifact: patchreport
+        - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+          parameters:
+            DisplayName: 'Publish Report Artifacts'
+            ArtifactPath: '$(Build.ArtifactStagingDirectory)'
+            ArtifactName: 'patchreport'
diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml
@@ -7,7 +7,9 @@ jobs:
       - template: /eng/pipelines/templates/variables/globals.yml
 
     pool:
-      name: azsdk-pool-mms-ubuntu-2004-general
+      name: $(LINUXPOOL)
+      vmImage: $(LINUXVMIMAGE)
+      os: linux
 
     steps:
       # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
@@ -105,16 +107,27 @@ jobs:
       - task: PowerShell@2
         inputs:
           filePath: $(System.DefaultWorkingDirectory)/eng/pipelines/scripts/Get-Linting-Reports.ps1
-          arguments: -OutputDirectory $(Build.ArtifactStagingDirectory)/linting-reports
+          arguments: -StagingDirectory $(Build.ArtifactStagingDirectory)
           pwsh: true
           workingDirectory: $(Pipeline.Workspace)
         displayName: 'Generate Linting Reports'
         condition: and(always(), ne(variables['LintingGoals'], ''))
 
-      - publish: $(Build.ArtifactStagingDirectory)/linting-reports.zip
-        displayName: 'Publish Linting Reports'
-        artifact: linting-reports-$(System.StageName)-$(System.JobName)-$(System.JobAttempt)
-        condition: and(always(), ne(variables['LintingGoals'], ''))
+      - pwsh: |
+          $artifactZip = "$(Build.ArtifactStagingDirectory)/linting-reports.zip"
+          $destinationDir = "$(Build.ArtifactStagingDirectory)/zipped-linting-reports"
+          if (Test-Path $artifactZip) {
+            New-Item -ItemType Directory -Path "$destinationDir"
+            Move-Item -Path $artifactZip -Destination "$destinationDir"
+          }
+        displayName: Move linting report zip to individual directory for artifact upload
+        condition: succeededOrFailed()
+
+      - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+        parameters:
+          DisplayName: 'Publish Linting Reports'
+          ArtifactPath: '$(Build.ArtifactStagingDirectory)/zipped-linting-reports'
+          ArtifactName: 'linting-reports'
 
       - pwsh: |
           if ((Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/linting-reports -Directory | Measure-Object).Count -gt 0) {

diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml
@@ -27,6 +27,8 @@ jobs:
           Repositories:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
+            - Name: azure-sdk/$(DocRepoName)
+              WorkingDirectory: $(DailyDocRepoLocation)
       # Pull and build the docker image.
       - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
         parameters:
@@ -94,17 +96,6 @@ jobs:
       - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml
         parameters:
           DailyBranchVariableName: DailyDocsBranchName
-      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-        parameters:
-          SkipCheckoutNone: true
-          Paths:
-            - package.json
-            - metadata/
-            - docs-ref-mapping/reference-unified.yml
-            - docs-ref-services/
-          Repositories:
-            - Name: azure-sdk/$(DocRepoName)
-              WorkingDirectory: $(DailyDocRepoLocation)
 
       # Docs daily updates is supposed to download packages from public feed repository, so we have to specify additional repositories in a POM or the profile.
       # Here is maven documentation: https://maven.apache.org/guides/mini/guide-multiple-repositories.html

diff --git a/eng/pipelines/latest-jdk.yml b/eng/pipelines/latest-jdk.yml
@@ -9,82 +9,94 @@ pr:
       - eng/pipelines/latest-jdk.yml
       - eng/scrips/Install-Latest-JDK.ps1
 
-variables:
-  - template: /eng/pipelines/templates/variables/globals.yml
-
-stages:
-  - stage: LatestJdk
-    displayName: Latest JDK
-
-    jobs:
-      - job: RunLatestJdk
-        displayName: Run Latest JDK
-        timeoutInMinutes: 180
-
-        pool:
-          name: 'azsdk-pool-mms-ubuntu-2004-general'
-          vmImage: 'MMSUbuntu20.04'
-
-        steps:
-          # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
-          # as we require the GitHub service connection to be loaded.
-          - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
-              - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+parameters:
+  - name: Artifacts
+    type: object
+    default:
+      - name: latest-jdk # fake artifact name
+        groupId: run # fake group id
+
+extends:
+  template: /eng/pipelines/templates/stages/1es-redirect.yml
+  parameters:
+    UseOfficial: false
+    stages:
+      - stage: LatestJdk
+        variables:
+          - template: /eng/pipelines/templates/variables/globals.yml
+          - template: /eng/pipelines/templates/variables/image.yml
+          - name: ArtifactsJson
+            value: '${{ convertToJson(parameters.Artifacts) }}'
+
+        displayName: Latest JDK
+
+        jobs:
+          - job: RunLatestJdk
+            displayName: Run Latest JDK
+            timeoutInMinutes: 180
+
+            pool:
+              name: $(LINUXPOOL)
+              image: $(LINUXVMIMAGE)
+              os: linux
+
+            steps:
+              # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
+              # as we require the GitHub service connection to be loaded.
+              - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}:
+                  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+                    parameters:
+                      Paths:
+                        - '**/*.xml'
+
+              - task: UsePythonVersion@0
+                displayName: 'Use Python $(PythonVersion)'
+                inputs:
+                  versionSpec: $(PythonVersion)
+
+              - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
+                parameters:
+                  JobType: 'LatestJdk'
+
+              - task: PythonScript@0
+                displayName: 'Generate FromSource POM and directories for sparse checkout'
+                inputs:
+                  scriptPath: 'eng/scripts/generate_from_source_pom.py'
+                  arguments: '--match-any-version --set-skip-linting-projects SkipLintingProjects --artifacts-list com.azure:azure-core'
+                  workingDirectory: '$(System.DefaultWorkingDirectory)'
+
+              - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml
+                parameters:
+                  SkipCheckoutNone: true
+                  Paths: $(SparseCheckoutDirectories)
+
+              - template: /eng/pipelines/templates/steps/install-reporting-tools.yml
+
+              - pwsh: |
+                  Write-Host "##vso[task.setvariable variable=IsLatestNonLtsJdk;]true"
+
+              - template: /eng/pipelines/templates/steps/install-latest-jdk.yml
                 parameters:
-                  Paths:
-                    - '**/*.xml'
-
-          - task: UsePythonVersion@0
-            displayName: 'Use Python $(PythonVersion)'
-            inputs:
-              versionSpec: $(PythonVersion)
-
-          - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
-            parameters:
-              JobType: 'LatestJdk'
-              Artifacts:
-                - name: latest-jdk # fake artifact name
-                  groupId: run # fake group id
-
-          - task: PythonScript@0
-            displayName: 'Generate FromSource POM and directories for sparse checkout'
-            inputs:
-              scriptPath: 'eng/scripts/generate_from_source_pom.py'
-              arguments: '--match-any-version --set-skip-linting-projects SkipLintingProjects --project-list com.azure:azure-core'
-              workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-          - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-            parameters:
-              SkipCheckoutNone: true
-              Paths: $(SparseCheckoutDirectories)
-
-          - template: /eng/pipelines/templates/steps/install-reporting-tools.yml
-
-          - pwsh: |
-              Write-Host "##vso[task.setvariable variable=IsLatestNonLtsJdk;]true"
-
-          - template: /eng/pipelines/templates/steps/install-latest-jdk.yml
-            parameters:
-              LatestJdkFeatureVersion: $(LatestNonLtsJdkFeatureVersion)
-
-          - task: Maven@3
-            displayName: 'Run All Libraries Latest JDK'
-            inputs:
-              mavenPomFile: ClientFromSourcePom.xml
-              options: '$(DefaultOptions) -T 1C -Dgpg.skip'
-              mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
-              javaHomeOption: 'JDKVersion'
-              jdkVersionOption: $(LatestNonLtsJavaVersion)
-              jdkArchitectureOption: 'x64'
-              publishJUnitResults: false
-              goals: 'clean install'
-            continueOnError: true
-
-          - task: PublishTestResults@2
-            condition: always()
-            inputs:
-              testResultsFormat: 'JUnit'
-              testResultsFiles: '**/TEST-*.xml'
-              searchFolder: '$(System.DefaultWorkingDirectory)/sdk'
-              mergeTestResults: true
-              testRunTitle: '$(Agent.JobName)_attempt_$(System.JobAttempt)'
+                  LatestJdkFeatureVersion: $(LatestNonLtsJdkFeatureVersion)
+
+              - task: Maven@3
+                displayName: 'Run All Libraries Latest JDK'
+                inputs:
+                  mavenPomFile: ClientFromSourcePom.xml
+                  options: '$(DefaultOptions) -T 1C -Dgpg.skip'
+                  mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
+                  javaHomeOption: 'JDKVersion'
+                  jdkVersionOption: $(LatestNonLtsJavaVersion)
+                  jdkArchitectureOption: 'x64'
+                  publishJUnitResults: false
+                  goals: 'clean install'
+                continueOnError: true
+
+              - task: PublishTestResults@2
+                condition: always()
+                inputs:
+                  testResultsFormat: 'JUnit'
+                  testResultsFiles: '**/TEST-*.xml'
+                  searchFolder: '$(System.DefaultWorkingDirectory)/sdk'
+                  mergeTestResults: true
+                  testRunTitle: '$(Agent.JobName)_attempt_$(System.JobAttempt)'
diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml
@@ -26,8 +26,9 @@ variables:
 jobs:
 - job: Signing
   pool:
-    name: azsdk-pool-mms-win-2022-general
-    vmImage: windows-2022
+    name: $(WINDOWSPOOL)
+    vmImage: $(WINDOWSVMIMAGE)
+    os: windows
   steps:
   - checkout: azure-sdk-build-tools
     path: azure-sdk-build-tools
@@ -50,8 +51,9 @@ jobs:
 - job: Release
   dependsOn: Signing
   pool:
-    name: azsdk-pool-mms-win-2022-general
-    vmImage: windows-2022
+    name: $(WINDOWSPOOL)
+    vmImage: $(WINDOWSVMIMAGE)
+    os: windows
   steps:
   - checkout: self
     path: azure-sdk-for-java

diff --git a/eng/pipelines/scripts/Get-Heap-Dump-Hprofs.ps1 b/eng/pipelines/scripts/Get-Heap-Dump-Hprofs.ps1
@@ -0,0 +1,32 @@
+<#
+.SYNOPSIS
+Captures any .hprof files in the build directory and moves them to a staging directory for artifact publishing.
+
+.DESCRIPTION
+This script is used to capture any .hprof files in the build directory and move them to a staging directory for
+artifact publishing. It also sets a pipeline variable to indicate whether any .hprof files were found.
+
+.PARAMETER StagingDirectory
+The directory where the .hprof files will be moved to.
+
+.PARAMETER OomArtifactName
+The name of the artifact to be created.
+#>
+
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$StagingDirectory,
+
+    [Parameter(Mandatory = $true)]
+    [string]$OomArtifactName
+)
+
+$hrpofs = Get-ChildItem -Path . -Recurse -Filter *.hprof -File
+
+if ($hrpofs.Count -gt 0) {
+    if (-not (Test-Path "$StagingDirectory/troubleshooting")) {
+        New-Item -ItemType Directory -Path "$StagingDirectory/troubleshooting" | Out-Null
+    }
+    Compress-Archive -Path $hrpofs -DestinationPath "$StagingDirectory/troubleshooting/$OomArtifactName.zip"
+    Write-Host "##vso[task.setvariable variable=HAS_TROUBLESHOOTING]true"
+}