Stress development maturity updates (#3481)

* Support local addons path override in stress test deployment

* Support username based deployId in local stress deployment

* Fix network chaos example

* Add stress test example release ci

* Support WhatIf in stress infrastructure provision script

* Fix image ref issues in powershell template

* Add stress cluster provision and validation pipeline

* Simplify provision pipeline yaml and enable sub config/sp auth

* Fix chart.lock local reference in stress example

* Simplify stress user detection

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Run helm plugin add with helper

* Use mms pool for stress test release

* Fix default parameter set for stress cluster provision script

* Add WhatIf support to ps module install helper function

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

eng/common/scripts/Helpers/PSModule-Helpers.ps1

@@ -48,8 +48,15 @@ function Update-PSModulePath()
 }
 
 # If we want to use another default repository other then PSGallery we can update the default parameters
-function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $DefaultPSRepositoryUrl)
+function Install-ModuleIfNotInstalled()
 {
+  [CmdletBinding(SupportsShouldProcess = $true)]
+  param(
+    [string]$moduleName,
+    [string]$version,
+    [string]$repositoryUrl = $DefaultPSRepositoryUrl
+  )
+
   # Check installed modules
   $modules = (Get-Module -ListAvailable $moduleName)
   if ($version -as [Version]) {
@@ -94,4 +101,4 @@ function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $D
   return $modules[0]
 }
 
-Update-PSModulePath
+Update-PSModulePath

eng/common/scripts/common.ps1

@@ -54,4 +54,4 @@ $GetOnboardedDocsMsPackagesForMonikerFn = "Get-${Language}-OnboardedDocsMsPackag
 $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
 $GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildrenForManagementPackages"
 $UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc"
-$GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"
+$GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"

eng/common/scripts/stress-testing/deploy-stress-tests.ps1

@@ -21,7 +21,10 @@ param(
     [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID),
 
     # Optional namespace override, otherwise the shell user or chart annotation will be used
-    [string]$Namespace
+    [string]$Namespace,
+
+    # Override remote stress-test-addons with local on-disk addons for development
+    [System.IO.FileInfo]$LocalAddonsPath
 )
 
 . $PSScriptRoot/stress-test-deployment-lib.ps1

eng/common/scripts/stress-testing/find-all-stress-packages.ps1

@@ -9,6 +9,7 @@ class StressTestPackageInfo {
     [string]$ReleaseName
     [string]$Dockerfile
     [string]$DockerBuildDir
+    [string]$Deployer
 }
 
 function FindStressPackages(
@@ -50,6 +51,17 @@ function MatchesAnnotations([hashtable]$chart, [hashtable]$filters) {
     return $true
 }
 
+function GetUsername() {
+    # Check GITHUB_USER for users in codespaces environments, since the default user is `codespaces` and
+    # we would like to avoid namespace overlaps for different codespaces users.
+    $stressUser = $env:GITHUB_USER ?? $env:USER ?? $env:USERNAME
+    # Remove spaces, underscores, etc. that may be in $namespace.
+    # Value must be a valid RFC 1123 DNS label: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
+    $stressUser = $stressUser -replace '_|\W', '-'
+
+    return $stressUser.ToLower()
+}
+
 function NewStressTestPackageInfo(
     [hashtable]$chart,
     [System.IO.FileInfo]$chartFile,
@@ -61,18 +73,7 @@ function NewStressTestPackageInfo(
     } elseif ($CI) {
         $chart.annotations.namespace
     } else {
-        # Check GITHUB_USER for users in codespaces environments, since the default user is `codespaces` and
-        # we would like to avoid namespace overlaps for different codespaces users.
-        $namespace = if ($env:GITHUB_USER) {
-            $env:GITHUB_USER
-        } elseif ($env:USER) {
-            $env:USER
-        } else {
-            $env:USERNAME
-        }
-        # Remove spaces, underscores, etc. that may be in $namespace. Value must be a valid RFC 1123 DNS label:
-        # https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
-        $namespace -replace '_|\W', '-'
+        GetUsername
     }
 
     return [StressTestPackageInfo]@{

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -67,11 +67,18 @@ function DeployStressTests(
     [string]$repository = '',
     [switch]$pushImages,
     [string]$clusterGroup = '',
-    [string]$deployId = 'local',
+    [string]$deployId = '',
     [switch]$login,
     [string]$subscription = '',
     [switch]$CI,
-    [string]$Namespace
+    [string]$Namespace,
+    [ValidateScript({
+        if (!(Test-Path $_)) {
+            throw "LocalAddonsPath $LocalAddonsPath does not exist"
+        }
+        return $true
+    })]
+    [System.IO.FileInfo]$LocalAddonsPath
 ) {
     if ($environment -eq 'test') {
         if ($clusterGroup -or $subscription) {
@@ -94,26 +101,37 @@ function DeployStressTests(
         Login -subscription $subscription -clusterGroup $clusterGroup -pushImages:$pushImages
     }
 
-    RunOrExitOnFailure helm repo add stress-test-charts https://stresstestcharts.blob.core.windows.net/helm/
+    $chartRepoName = 'stress-test-charts'
+    if ($LocalAddonsPath) {
+        $absAddonsPath = Resolve-Path $LocalAddonsPath
+        if (!(helm plugin list | Select-String 'file')) {
+            RunOrExitOnFailure helm plugin add (Join-Path $absAddonsPath file-plugin)
+        }
+        RunOrExitOnFailure helm repo add --force-update $chartRepoName file://$absAddonsPath
+    } else {
+        RunOrExitOnFailure helm repo add --force-update $chartRepoName https://stresstestcharts.blob.core.windows.net/helm/
+    }
+
     Run helm repo update
     if ($LASTEXITCODE) { return $LASTEXITCODE }
 
+    $deployer = if ($deployId) { $deployId } else { GetUsername }
     $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace
     Write-Host "" "Found $($pkgs.Length) stress test packages:"
     Write-Host $pkgs.Directory ""
     foreach ($pkg in $pkgs) {
         Write-Host "Deploying stress test at '$($pkg.Directory)'"
         DeployStressPackage `
             -pkg $pkg `
-            -deployId $deployId `
+            -deployId $deployer `
             -environment $environment `
             -repositoryBase $repository `
             -pushImages:$pushImages `
             -login:$login
     }
 
-    Write-Host "Releases deployed by $deployId"
-    Run helm list --all-namespaces -l deployId=$deployId
+    Write-Host "Releases deployed by $deployer"
+    Run helm list --all-namespaces -l deployId=$deployer
 
     if ($FailedCommands) {
         Write-Warning "The following commands failed:"

eng/pipelines/stress-cluster-provision.yml

@@ -0,0 +1,20 @@
+pr: none
+
+trigger: none
+
+parameters:
+  - name: Environment
+    type: string
+    default: test
+    values:
+    - prod
+    - test
+  - name: WhatIf
+    type: boolean
+    default: false
+
+extends:
+  template: /eng/pipelines/templates/jobs/stress-cluster-provision.yml
+  parameters:
+    Environment: ${{ parameters.Environment }}
+    WhatIf: ${{ parameters.WhatIf }}

eng/pipelines/stress-test-release.yml

@@ -25,69 +25,9 @@ parameters:
     type: string
     default: main
 
-variables:
-- template: /eng/pipelines/templates/variables/globals.yml
-
-jobs:
-- job:
-  strategy:
-    matrix:
-      ${{ if or(eq(parameters.TestRepository, 'examples'), eq(parameters.TestRepository, 'all')) }}:
-        examples:
-          Repository: Azure/azure-sdk-tools
-          Filters: '@{ "example" = "true" }'
-      ${{ if or(eq(parameters.TestRepository, 'javascript'), eq(parameters.TestRepository, 'all')) }}:
-        javascript:
-          Repository: Azure/azure-sdk-for-js
-          Filters: '@{}'
-      ${{ if or(eq(parameters.TestRepository, 'java'), eq(parameters.TestRepository, 'all')) }}:
-        java:
-          Repository: Azure/azure-sdk-for-java
-          Filters: '@{}'
-      ${{ if or(eq(parameters.TestRepository, 'net'), eq(parameters.TestRepository, 'all')) }}:
-        net:
-          Repository: Azure/azure-sdk-for-net
-          Filters: '@{}'
-      ${{ if or(eq(parameters.TestRepository, 'python'), eq(parameters.TestRepository, 'all')) }}:
-        python:
-          Repository: Azure/azure-sdk-for-python
-          Filters: '@{}'
-      ${{ if or(eq(parameters.TestRepository, 'go'), eq(parameters.TestRepository, 'all')) }}:
-        go:
-          Repository: Azure/azure-sdk-for-go
-          Filters: '@{}'
-  pool:
-    vmImage: 'ubuntu-20.04'
-    #name: 'azsdk-pool-mms-ubuntu-2004-general'
-    #vmImage: 'MMSUbuntu20.04'
-  steps:
-    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-      parameters:
-        Repositories:
-          - Name: $(Repository)
-            Commitish: ${{ parameters.DeployFromBranchOrCommit }}
-            WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Repository)
-        Paths:
-          - '/*'
-          - '!sdk/**/test-recordings'
-          - '!sdk/**/session-records'
-          - '!sdk/**/SessionRecords'
-
-    - task: AzureCLI@2
-      displayName: Build and Deploy Stress Tests
-      inputs:
-        ${{ if eq(parameters.Environment, 'prod') }}:
-          azureSubscription: Azure SDK Test Resources
-        ${{ if eq(parameters.Environment, 'test') }}:
-          azureSubscription: Azure SDK Developer Playground
-        scriptType: pscore
-        scriptPath: $(System.DefaultWorkingDirectory)/$(Repository)/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
-        arguments:
-          -SearchDirectory '$(System.DefaultWorkingDirectory)/$(Repository)'
-          -Filters $(Filters)
-          -Environment '${{ parameters.Environment }}'
-          -Repository '$(Agent.JobName)'
-          -PushImages
-          -Login
-          -DeployId '$(Build.BuildNumber)'
-          -CI
+extends:
+  template: /eng/pipelines/templates/jobs/stress-test-release.yml
+  parameters:
+    Environment: ${{ parameters.Environment }}
+    TestRepository: ${{ parameters.TestRepository }}
+    DeployFromBranchOrCommit: ${{ parameters.DeployFromBranchOrCommit }}

eng/pipelines/templates/jobs/stress-cluster-provision.yml

@@ -0,0 +1,28 @@
+parameters:
+  - name: Environment
+    type: string
+  - name: WhatIf
+    type: boolean
+    default: true
+
+jobs:
+  - job:
+    variables:
+      - template: /eng/pipelines/templates/variables/globals.yml
+    ${{ if eq(parameters.WhatIf, true) }}:
+      displayName: 'Validate Provision'
+    ${{ else }}:
+      displayName: 'Provision'
+    pool:
+      name: 'azsdk-pool-mms-ubuntu-2004-general'
+      vmImage: 'MMSUbuntu20.04'
+    steps:
+      - pwsh: |
+          $subscriptionConfiguration = @'
+            $(sub-config-azure-cloud-test-resources)
+          '@ | ConvertFrom-Json -AsHashtable;
+
+          tools/stress-cluster/cluster/provision.ps1 `
+            -WhatIf:$${{ parameters.WhatIf }} `
+            -Environment ${{ parameters.Environment }} `
+            @subscriptionConfiguration

eng/pipelines/templates/jobs/stress-test-release.yml

@@ -0,0 +1,76 @@
+parameters:
+  - name: Environment
+    type: string
+    default: test
+  - name: TestRepository
+    type: string
+    default: all
+  - name: DeployFromBranchOrCommit
+    type: string
+    default: main
+
+jobs:
+- job:
+  variables:
+    - template: /eng/pipelines/templates/variables/globals.yml
+  strategy:
+    matrix:
+      ${{ if or(eq(parameters.TestRepository, 'examples'), eq(parameters.TestRepository, 'all')) }}:
+        examples:
+          Repository: Azure/azure-sdk-tools
+          Filters: '@{ "example" = "true" }'
+      ${{ if or(eq(parameters.TestRepository, 'javascript'), eq(parameters.TestRepository, 'all')) }}:
+        javascript:
+          Repository: Azure/azure-sdk-for-js
+          Filters: '@{}'
+      ${{ if or(eq(parameters.TestRepository, 'java'), eq(parameters.TestRepository, 'all')) }}:
+        java:
+          Repository: Azure/azure-sdk-for-java
+          Filters: '@{}'
+      ${{ if or(eq(parameters.TestRepository, 'net'), eq(parameters.TestRepository, 'all')) }}:
+        net:
+          Repository: Azure/azure-sdk-for-net
+          Filters: '@{}'
+      ${{ if or(eq(parameters.TestRepository, 'python'), eq(parameters.TestRepository, 'all')) }}:
+        python:
+          Repository: Azure/azure-sdk-for-python
+          Filters: '@{}'
+      ${{ if or(eq(parameters.TestRepository, 'go'), eq(parameters.TestRepository, 'all')) }}:
+        go:
+          Repository: Azure/azure-sdk-for-go
+          Filters: '@{}'
+  pool:
+    name: 'azsdk-pool-mms-ubuntu-2004-general'
+    vmImage: 'MMSUbuntu20.04'
+  steps:
+    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      parameters:
+        Repositories:
+          - Name: $(Repository)
+            Commitish: ${{ parameters.DeployFromBranchOrCommit }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Repository)
+        Paths:
+          - '/*'
+          - '!sdk/**/recordings/*'
+          - '!sdk/**/test-recordings/*'
+          - '!sdk/**/session-records/*'
+          - '!sdk/**/SessionRecords/*'
+
+    - task: AzureCLI@2
+      displayName: Build and Deploy Stress Tests
+      inputs:
+        ${{ if eq(parameters.Environment, 'prod') }}:
+          azureSubscription: Azure SDK Test Resources
+        ${{ if eq(parameters.Environment, 'test') }}:
+          azureSubscription: Azure SDK Playground
+        scriptType: pscore
+        scriptPath: $(System.DefaultWorkingDirectory)/$(Repository)/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
+        arguments:
+          -SearchDirectory '$(System.DefaultWorkingDirectory)/$(Repository)'
+          -Filters $(Filters)
+          -Environment '${{ parameters.Environment }}'
+          -Repository '$(Agent.JobName)'
+          -PushImages
+          -Login
+          -DeployId '$(Build.BuildNumber)'
+          -CI

eng/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml

@@ -22,11 +22,11 @@ stages:
         strategy:
           matrix:
             Windows:
-              Pool: 'azsdk-pool-mms-ubuntu-2004-general'
-              vmImage: 'MMSUbuntu20.04'
-            Linux:
               Pool: 'azsdk-pool-mms-win-2019-general'
-              vmImage: 'MMS2019'
+              Image: 'MMS2019'
+            Linux:
+              Pool: 'azsdk-pool-mms-ubuntu-2004-general'
+              Image: 'MMSUbuntu20.04'
             Mac:
               Pool: 'Azure Pipelines'
               Image: 'macOS-10.15'
@@ -48,7 +48,7 @@ stages:
                 $config = New-PesterConfiguration
                 $config.CodeCoverage.Enabled = $true
                 $config.TestResult.Enabled = $true
-                
+
                 if ($tags) {
                   $config.Filter.Tag = $tags
                 }
@@ -75,4 +75,4 @@ stages:
             inputs:
               codeCoverageTool: 'JaCoCo'
               summaryFileLocation: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/coverage.xml'
-              pathToSources: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}'
+              pathToSources: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}'

tools/stress-cluster/chaos/examples/network-stress-example/Chart.yaml

@@ -11,4 +11,4 @@ annotations:
 dependencies:
 - name: stress-test-addons
   version: 0.1.17
-  repository: https://stresstestcharts.blob.core.windows.net/helm/
+  repository: "@stress-test-charts"

tools/stress-cluster/chaos/examples/network-stress-example/templates/network_loss.yaml

@@ -1,4 +1,4 @@
-{{- include "stress-test-addons.chaos-wrapper.tpl" (list . "network-chaos") -}}
+{{- include "stress-test-addons.chaos-wrapper.tpl" (list . "stress.network-chaos") -}}
 {{- define "stress.network-chaos" -}}
 apiVersion: chaos-mesh.org/v1alpha1
 kind: NetworkChaos