Handle cleanup for leased and encrypted storage accounts. Reduce thro…

…ttling (#9212)
Azure · Oct 18, 2024 · 08a486a · 08a486a
1 parent 4554792
commit 08a486a
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 16 deletions.
diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -247,19 +247,40 @@ function Remove-WormStorageAccounts() {
         if (!$hasContainers) { continue }
 
         $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName
+        $containers = $ctx | Get-AzStorageContainer
+        $blobs = $containers | Get-AzStorageBlob
 
-        $immutableBlobs = $ctx `
-        | Get-AzStorageContainer `
+        $immutableBlobs = $containers `
         | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } `
         | Get-AzStorageBlob
         try {
           foreach ($blob in $immutableBlobs) {
-            Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)"
-            $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null
+            # We can't edit blobs with customer encryption without using that key
+            # so just try to delete them fully instead. It is unlikely they
+            # will also have a legal hold enabled.
+            if (($blob | Get-Member 'ListBlobProperties') `
+                -and $blob.ListBlobProperties.Properties.CustomerProvidedKeySha256) {
+              Write-Host "Removing customer encrypted blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)"
+              $blob | Remove-AzStorageBlob -Force
+              continue
+            }
+
+            if (!($blob | Get-Member 'BlobProperties')) {
+              continue
+            }
+
+            if ($blob.BlobProperties.LeaseState -eq 'Leased') {
+              Write-Host "Breaking blob lease: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)"
+              $blob.ICloudBlob.BreakLease()
+            }
+
+            if ($blob.BlobProperties.HasLegalHold) {
+              Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)"
+              $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null
+            }
           }
-        }
-        catch {
-          Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group"
+        } catch {
+          Write-Warning "Ensure user has 'Storage Blob Data Owner' RBAC permission on subscription or resource group"
           Write-Error $_
           throw
         }
@@ -273,13 +294,19 @@ function Remove-WormStorageAccounts() {
           }
 
           try {
-            Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)"
-            $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy
+            foreach ($blob in $blobs) {
+              if ($blob.BlobProperties.ImmutabilityPolicy.PolicyMode) {
+                Write-Host "Removing immutability policy - blob: $($blob.Name), account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)"
+                $null = $blob | Remove-AzStorageBlobImmutabilityPolicy
+              }
+            }
           }
           catch {}
 
           try {
-            $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force
+            foreach ($blob in $blobs) {
+              $blob | Remove-AzStorageBlob -Force
+            }
             $succeeded = $true
           }
           catch {
@@ -290,9 +317,8 @@ function Remove-WormStorageAccounts() {
 
         try {
           # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs
-          $ctx | Get-AzStorageContainer | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force }
-        }
-        catch {
+          $containers | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force }
+        } catch {
           Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account."
           Write-Warning $_
         }

diff --git a/eng/pipelines/live-test-cleanup-template.yml b/eng/pipelines/live-test-cleanup-template.yml
@@ -2,6 +2,9 @@ parameters:
   - name: DryRun
     type: boolean
     default: false
+  - name: GroupFilter
+    type: string
+    default: '*'
   - name: DisplayName
     type: string
   - name: ServiceConnection
@@ -87,7 +90,8 @@ steps:
           @subscriptionConfiguration `
           -Verbose `
           ${{ parameters.AdditionalParameters }} `
-          -WhatIf:$${{ parameters.DryRun }}
+          -WhatIf:$${{ parameters.DryRun }} `
+          -GroupFilter '${{ parameters.GroupFilter }}'
       displayName: ${{ parameters.DisplayName }}
       continueOnError: true
       env:

diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml
@@ -5,6 +5,9 @@ parameters:
   - name: DryRun
     type: boolean
     default: false
+  - name: GroupFilter
+    type: string
+    default: '*'
   - name: Pool
     type: string
     default: azsdk-pool-mms-ubuntu-2204-general
@@ -97,6 +100,7 @@ stages:
       - template: ./live-test-cleanup-template.yml
         parameters:
           DryRun: ${{ parameters.DryRun }}
+          GroupFilter: ${{ parameters.GroupFilter }}
           DisplayName: ${{ subscription.DisplayName }}
           ServiceConnection: ${{ subscription.ServiceConnection }}
           SubscriptionConfigurationFilePaths: ${{ subscription.SubscriptionConfigurationFilePaths }}

diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
@@ -55,6 +55,8 @@ param (
   [Parameter()]
   [string] $AllowListPath = "$PSScriptRoot/cleanup-allowlist.txt",
 
+  [string] $GroupFilter = '*',
+
   [Parameter()]
   [switch] $Force,
 
@@ -351,7 +353,7 @@ function DeleteOrUpdateResourceGroups() {
   }
 
   Write-Verbose "Fetching groups"
-  [Array]$allGroups = Retry { Get-AzResourceGroup }
+  [Array]$allGroups = Retry { Get-AzResourceGroup } | Where-Object { $_.ResourceGroupName -like $GroupFilter }
   $toDelete = @()
   $toClean = @()
   $toDeleteSoon = @()
@@ -449,7 +451,7 @@ function DeleteAndPurgeGroups([array]$toDelete) {
   }
 
   if (!$purgeableResources.Count) {
-    return
+    return $hasError
   }
   if ($Force -or $PSCmdlet.ShouldProcess("Purgable Resources", "Delete Purgeable Resources")) {
     # Purge all the purgeable resources and get a list of resources (as a collection) we need to follow-up on.