From 08a486a5dfc920cac3728ae5cb54ccd091edc875 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 18 Oct 2024 10:55:36 -0400 Subject: [PATCH] Handle cleanup for leased and encrypted storage accounts. Reduce throttling (#9212) --- .../scripts/Helpers/Resource-Helpers.ps1 | 52 ++++++++++++++----- eng/pipelines/live-test-cleanup-template.yml | 6 ++- eng/pipelines/live-test-cleanup.yml | 4 ++ eng/scripts/live-test-resource-cleanup.ps1 | 6 ++- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index fdbb44186ea..a5806ea1ea0 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -247,19 +247,40 @@ function Remove-WormStorageAccounts() { if (!$hasContainers) { continue } $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName + $containers = $ctx | Get-AzStorageContainer + $blobs = $containers | Get-AzStorageBlob - $immutableBlobs = $ctx ` - | Get-AzStorageContainer ` + $immutableBlobs = $containers ` | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } ` | Get-AzStorageBlob try { foreach ($blob in $immutableBlobs) { - Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" - $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null + # We can't edit blobs with customer encryption without using that key + # so just try to delete them fully instead. It is unlikely they + # will also have a legal hold enabled. + if (($blob | Get-Member 'ListBlobProperties') ` + -and $blob.ListBlobProperties.Properties.CustomerProvidedKeySha256) { + Write-Host "Removing customer encrypted blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob | Remove-AzStorageBlob -Force + continue + } + + if (!($blob | Get-Member 'BlobProperties')) { + continue + } + + if ($blob.BlobProperties.LeaseState -eq 'Leased') { + Write-Host "Breaking blob lease: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob.ICloudBlob.BreakLease() + } + + if ($blob.BlobProperties.HasLegalHold) { + Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null + } } - } - catch { - Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group" + } catch { + Write-Warning "Ensure user has 'Storage Blob Data Owner' RBAC permission on subscription or resource group" Write-Error $_ throw } @@ -273,13 +294,19 @@ function Remove-WormStorageAccounts() { } try { - Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" - $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy + foreach ($blob in $blobs) { + if ($blob.BlobProperties.ImmutabilityPolicy.PolicyMode) { + Write-Host "Removing immutability policy - blob: $($blob.Name), account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + $null = $blob | Remove-AzStorageBlobImmutabilityPolicy + } + } } catch {} try { - $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + foreach ($blob in $blobs) { + $blob | Remove-AzStorageBlob -Force + } $succeeded = $true } catch { @@ -290,9 +317,8 @@ function Remove-WormStorageAccounts() { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs - $ctx | Get-AzStorageContainer | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } - } - catch { + $containers | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } + } catch { Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account." Write-Warning $_ } diff --git a/eng/pipelines/live-test-cleanup-template.yml b/eng/pipelines/live-test-cleanup-template.yml index aa9c1d47b57..afac7c6d730 100644 --- a/eng/pipelines/live-test-cleanup-template.yml +++ b/eng/pipelines/live-test-cleanup-template.yml @@ -2,6 +2,9 @@ parameters: - name: DryRun type: boolean default: false + - name: GroupFilter + type: string + default: '*' - name: DisplayName type: string - name: ServiceConnection @@ -87,7 +90,8 @@ steps: @subscriptionConfiguration ` -Verbose ` ${{ parameters.AdditionalParameters }} ` - -WhatIf:$${{ parameters.DryRun }} + -WhatIf:$${{ parameters.DryRun }} ` + -GroupFilter '${{ parameters.GroupFilter }}' displayName: ${{ parameters.DisplayName }} continueOnError: true env: diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml index d28f77b646a..6359d99fcb4 100644 --- a/eng/pipelines/live-test-cleanup.yml +++ b/eng/pipelines/live-test-cleanup.yml @@ -5,6 +5,9 @@ parameters: - name: DryRun type: boolean default: false + - name: GroupFilter + type: string + default: '*' - name: Pool type: string default: azsdk-pool-mms-ubuntu-2204-general @@ -97,6 +100,7 @@ stages: - template: ./live-test-cleanup-template.yml parameters: DryRun: ${{ parameters.DryRun }} + GroupFilter: ${{ parameters.GroupFilter }} DisplayName: ${{ subscription.DisplayName }} ServiceConnection: ${{ subscription.ServiceConnection }} SubscriptionConfigurationFilePaths: ${{ subscription.SubscriptionConfigurationFilePaths }} diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index 2bdbc8425c7..6122319adad 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -55,6 +55,8 @@ param ( [Parameter()] [string] $AllowListPath = "$PSScriptRoot/cleanup-allowlist.txt", + [string] $GroupFilter = '*', + [Parameter()] [switch] $Force, @@ -351,7 +353,7 @@ function DeleteOrUpdateResourceGroups() { } Write-Verbose "Fetching groups" - [Array]$allGroups = Retry { Get-AzResourceGroup } + [Array]$allGroups = Retry { Get-AzResourceGroup } | Where-Object { $_.ResourceGroupName -like $GroupFilter } $toDelete = @() $toClean = @() $toDeleteSoon = @() @@ -449,7 +451,7 @@ function DeleteAndPurgeGroups([array]$toDelete) { } if (!$purgeableResources.Count) { - return + return $hasError } if ($Force -or $PSCmdlet.ShouldProcess("Purgable Resources", "Delete Purgeable Resources")) { # Purge all the purgeable resources and get a list of resources (as a collection) we need to follow-up on.