From df8d32f43f27e6112bf48bb3b3c4512b6a297df4 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 14:49:26 -0700
Subject: [PATCH 01/11] Attempt to purge all vaults, managed HSMs

Reverts #1910. Vaults and managed HSMs are automatically purged on their purge date. The point was to purge them daily to preserve capacity. The default purge date is +90 days.
---
 .../scripts/Helpers/Resource-Helpers.ps1      | 46 ++++++++++---------
 eng/pipelines/live-test-cleanup.yml           |  1 +
 eng/scripts/live-test-resource-cleanup.ps1    | 13 +-----
 3 files changed, 27 insertions(+), 33 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 188639d46d4..97ecf1c1222 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -7,6 +7,18 @@ function Get-PurgeableGroupResources {
   )
   $purgeableResources = @()
 
+  # Discover Managed HSMs first since they are a premium resource.
+  Write-Verbose "Retrieving deleted Managed HSMs from resource group $ResourceGroupName"
+
+  # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
+  $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
+
+  if ($deletedHsms) {
+    Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
+    $purgeableResources += $deletedHsms
+  }
+
   Write-Verbose "Retrieving deleted Key Vaults from resource group $ResourceGroupName"
 
   # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled.
@@ -21,34 +33,13 @@ function Get-PurgeableGroupResources {
     $purgeableResources += $deletedKeyVaults
   }
 
-  Write-Verbose "Retrieving deleted Managed HSMs from resource group $ResourceGroupName"
-
-  # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
-  $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
-
-  if ($deletedHsms) {
-    Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
-    $purgeableResources += $deletedHsms
-  }
-
   return $purgeableResources
 }
 function Get-PurgeableResources {
   $purgeableResources = @()
   $subscriptionId = (Get-AzContext).Subscription.Id
 
-  Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
-
-  # Get deleted Key Vaults for the current subscription.
-  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
-
-  if ($deletedKeyVaults) {
-    Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
-    $purgeableResources += $deletedKeyVaults
-  }
-
+  # Discover Managed HSMs first since they are a premium resource.
   Write-Verbose "Retrieving deleted Managed HSMs from subscription $subscriptionId"
 
   # Get deleted Managed HSMs for the current subscription.
@@ -75,6 +66,17 @@ function Get-PurgeableResources {
     }
   }
 
+  Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
+
+  # Get deleted Key Vaults for the current subscription.
+  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+
+  if ($deletedKeyVaults) {
+    Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
+    $purgeableResources += $deletedKeyVaults
+  }
+
   return $purgeableResources
 }
 
diff --git a/eng/pipelines/live-test-cleanup.yml b/eng/pipelines/live-test-cleanup.yml
index 55861263473..9618154f90f 100644
--- a/eng/pipelines/live-test-cleanup.yml
+++ b/eng/pipelines/live-test-cleanup.yml
@@ -9,6 +9,7 @@ stages:
 
   jobs:
   - job: Run
+    timeoutInMinutes: 0
     pool:
       vmImage: ubuntu-20.04
 
diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index 8f3c7763ca2..01f51345f35 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -67,7 +67,8 @@ Write-Host "Count $($hasDeleteAfter.Count)"
 $toDelete = $hasDeleteAfter.Where({ $deleteDate = ($_.Tags.DeleteAfter -as [DateTime]); (!$deleteDate -or $now -gt $deleteDate) })
 Write-Host "Groups to delete: $($toDelete.Count)"
 
-$purgeableResources = @()
+# Get purgeable resources arleady in a deleted state.
+$purgeableResources = Get-PurgeableResources
 
 foreach ($rg in $toDelete)
 {
@@ -85,16 +86,6 @@ foreach ($rg in $toDelete)
   }
 }
 
-# Get purgeable resources already in a deleted state coerced into a collection even if empty.
-$purgeableResources = Get-PurgeableResources
-$allPurgeCount = $purgeableResources.Count
-
-# Filter down to the ones that we can actually perge.
-$purgeableResources = $purgeableResources.Where({ $purgeDate = $_.ScheduledPurgeDate -as [DateTime]; (!$purgeDate -or $now -gt $purgeDate) })
-
 # Purge all the purgeable resources.
 Write-Host "Attempting to purge $($purgeableResources.Count) resources."
-if ($allPurgeCount -gt $purgeableResources.Count) {
-  Write-Host "Skipping $($allPurgeCount - $purgeableResources.Count) as their purge date is still in the future."
-}
 Remove-PurgeableResources $purgeableResources

From a2cd5b346bafe50eb103cdc1be84ce6d4f5ab526 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 17:39:41 -0700
Subject: [PATCH 02/11] Add timeout and more logging

---
 .../scripts/Helpers/Resource-Helpers.ps1      | 74 ++++++++++++++-----
 eng/scripts/live-test-resource-cleanup.ps1    |  8 +-
 2 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 97ecf1c1222..b3dba47e001 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -12,7 +12,8 @@ function Get-PurgeableGroupResources {
 
   # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
   $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru `
+    | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
 
   if ($deletedHsms) {
     Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
@@ -25,8 +26,9 @@ function Get-PurgeableGroupResources {
   $deletedKeyVaults = Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object {
     # Enumerating vaults from a resource group does not return all properties we required.
     Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } `
-      | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
-  }
+      | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
+      | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
+    }
 
   if ($deletedKeyVaults) {
     Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
@@ -35,6 +37,7 @@ function Get-PurgeableGroupResources {
 
   return $purgeableResources
 }
+
 function Get-PurgeableResources {
   $purgeableResources = @()
   $subscriptionId = (Get-AzContext).Subscription.Id
@@ -51,6 +54,7 @@ function Get-PurgeableResources {
     foreach ($r in $content.value) {
       $deletedHsms += [pscustomobject] @{
         AzsdkResourceType = 'Managed HSM'
+        AzsdkName = $r.name
         Id = $r.id
         Name = $r.name
         Location = $r.properties.location
@@ -70,7 +74,8 @@ function Get-PurgeableResources {
 
   # Get deleted Key Vaults for the current subscription.
   $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
+    | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
 
   if ($deletedKeyVaults) {
     Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
@@ -85,7 +90,14 @@ function Get-PurgeableResources {
 filter Remove-PurgeableResources {
   param (
     [Parameter(Position=0, ValueFromPipeline=$true)]
-    [object[]] $Resource
+    [object[]] $Resource,
+
+    [Parameter()]
+    [ValidateRange(1, [int]::MaxValue)]
+    [int] $Timeout = 30,
+
+    [Parameter()]
+    [switch] $PassThru
   )
 
   if (!$Resource) {
@@ -95,38 +107,39 @@ filter Remove-PurgeableResources {
   $subscriptionId = (Get-AzContext).Subscription.Id
 
   foreach ($r in $Resource) {
+    Log "Attempting to purge $($r.AzsdkResourceType) '$($r.AzsdkName)'"
     switch ($r.AzsdkResourceType) {
       'Key Vault' {
-        Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'"
         if ($r.EnablePurgeProtection) {
           # We will try anyway but will ignore errors
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue
+        Wait-PurgeableResource { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue } -Timeout:$Timeout -PassThru:$PassThru
       }
 
       'Managed HSM' {
-        Log "Attempting to purge $($r.AzsdkResourceType) '$($r.Name)'"
         if ($r.EnablePurgeProtection) {
           # We will try anyway but will ignore errors
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
-        if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-          Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
-        } elseif ($response.Content) {
-          $content = $response.Content | ConvertFrom-Json
-          if ($content.error) {
-            $err = $content.error
-            Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+        Wait-PurgeableResource -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
+          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
+          if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+            Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+          } elseif ($response.Content) {
+            $content = $response.Content | ConvertFrom-Json
+            if ($content.error) {
+              $err = $content.error
+              Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+            }
           }
         }
       }
 
       default {
-        Write-Warning "Cannot purge resource type $($r.AzsdkResourceType). Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
+        Write-Warning "Cannot purge $($r.AzsdkResourceType) '$($r.AzsdkName)'. Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
       }
     }
   }
@@ -136,3 +149,30 @@ filter Remove-PurgeableResources {
 function Log($Message) {
   Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
+
+function Wait-PurgeableResource {
+  param (
+    [Parameter(Mandatory=$true, Position=0)]
+    [scriptblock] $ScriptBlock,
+
+    [Parameter(Mandatory=$true, Position=1)]
+    [object] $Resource,
+
+    [Parameter()]
+    [ValidateRange(1, [int]::MaxValue)]
+    [int] $Timeout = 30,
+
+    [Parameter()]
+    [switch] $PassThru
+  )
+
+  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
+  $null = Wait-Job -Job $j -Timeout $Timeout
+
+  if ($j.State -eq 'Running') {
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$(Resource.AzsdkName)': {$ScriptBlock}"
+    if ($PassThru) {
+      $Resource
+    }
+  }
+}
diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index 01f51345f35..e06c07f82a5 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -86,6 +86,10 @@ foreach ($rg in $toDelete)
   }
 }
 
-# Purge all the purgeable resources.
+# Purge all the purgeable resources and get a list of resources (as a collection) we need to follow-up on.
 Write-Host "Attempting to purge $($purgeableResources.Count) resources."
-Remove-PurgeableResources $purgeableResources
+$failedResources = @(Remove-PurgeableResources $purgeableResources -PassThru)
+if ($failedResources) {
+  Write-Host "Failed to delete the following $($failedResources.Count) resources:"
+  $failedResources | Sort-Object AzsdkResourceType, AzsdkName | Format-Table -Property @{l='Type'; e={$_.AzsdkResourceType}}, @{l='Name'; e={$_.AzsdkName}}
+}

From 8e49250b3d30c1a766a0e99da64f2067cd20505b Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 17:40:39 -0700
Subject: [PATCH 03/11] Fix typo in comment

---
 eng/scripts/live-test-resource-cleanup.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index e06c07f82a5..94a14580e80 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -67,7 +67,7 @@ Write-Host "Count $($hasDeleteAfter.Count)"
 $toDelete = $hasDeleteAfter.Where({ $deleteDate = ($_.Tags.DeleteAfter -as [DateTime]); (!$deleteDate -or $now -gt $deleteDate) })
 Write-Host "Groups to delete: $($toDelete.Count)"
 
-# Get purgeable resources arleady in a deleted state.
+# Get purgeable resources already in a deleted state.
 $purgeableResources = Get-PurgeableResources
 
 foreach ($rg in $toDelete)

From c0f166543c1912ad887dced9f644d61d4bb7c56d Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 17:53:09 -0700
Subject: [PATCH 04/11] Pass required -Resource

---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index b3dba47e001..dbffde230ae 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -115,7 +115,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue } -Timeout:$Timeout -PassThru:$PassThru
+        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
       }
 
       'Managed HSM' {
@@ -124,7 +124,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
+        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
           $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
           if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
             Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."

From 98bdcc71083ba8dc6ed8b829ebb3071590cea599 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 18:31:59 -0700
Subject: [PATCH 05/11] Fix log message

---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 2 +-
 eng/scripts/live-test-resource-cleanup.ps1      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index dbffde230ae..e064f1b6831 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -170,7 +170,7 @@ function Wait-PurgeableResource {
   $null = Wait-Job -Job $j -Timeout $Timeout
 
   if ($j.State -eq 'Running') {
-    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$(Resource.AzsdkName)': {$ScriptBlock}"
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)': {$ScriptBlock}"
     if ($PassThru) {
       $Resource
     }
diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index 94a14580e80..582166525f6 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -90,6 +90,6 @@ foreach ($rg in $toDelete)
 Write-Host "Attempting to purge $($purgeableResources.Count) resources."
 $failedResources = @(Remove-PurgeableResources $purgeableResources -PassThru)
 if ($failedResources) {
-  Write-Host "Failed to delete the following $($failedResources.Count) resources:"
+  Write-Warning "Failed to delete the following $($failedResources.Count) resources:"
   $failedResources | Sort-Object AzsdkResourceType, AzsdkName | Format-Table -Property @{l='Type'; e={$_.AzsdkResourceType}}, @{l='Name'; e={$_.AzsdkName}}
 }

From 9e3fb0375d7559253f28882c5bc8eee94288589f Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 21:26:45 -0700
Subject: [PATCH 06/11] Ensure the $Resource is correctly captured

Added comment to new code explaining why, since ScriptBlock.GetNewClosure() is not working as expected.
---
 .../scripts/Helpers/Resource-Helpers.ps1      | 26 +++++++++++++------
 eng/scripts/live-test-resource-cleanup.ps1    |  2 +-
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index e064f1b6831..8e7a4e701fb 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -73,7 +73,7 @@ function Get-PurgeableResources {
   Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
 
   # Get deleted Key Vaults for the current subscription.
-  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
+  $deletedKeyVaults = Get-AzKeyVault -InRemovedState `
     | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
     | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
 
@@ -115,7 +115,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
       }
 
       'Managed HSM' {
@@ -124,7 +124,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
           $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
           if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
             Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
@@ -152,11 +152,11 @@ function Log($Message) {
 
 function Wait-PurgeableResource {
   param (
-    [Parameter(Mandatory=$true, Position=0)]
+    [Parameter(Mandatory=$true)]
     [scriptblock] $ScriptBlock,
 
-    [Parameter(Mandatory=$true, Position=1)]
-    [object] $Resource,
+    [Parameter(Mandatory=$true)]
+    $Resource,
 
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
@@ -166,13 +166,23 @@ function Wait-PurgeableResource {
     [switch] $PassThru
   )
 
-  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
+  # We build a new AST to pass the `$Resource` as `$r`, which is the `foreach` variable declared in `Remove-PurgeableResources` above.
+  # This is done to make writing a purge script feel natural without having to worry about scope since `[ScriptBlock].GetNewClosure()`
+  # does not capture `$r` appropriately. If the variable name in the `foreach` above is changed, it must be changed here as well.
+  $scriptBlockAst = [System.Management.Automation.Language.Parser]::ParseInput(@"
+  param (`$r)
+  $($ScriptBlock.Ast.EndBlock.ToString())
+"@, [ref] $null, [ref] $null)
+
+  $j = Start-ThreadJob -ScriptBlock $scriptBlockAst.GetScriptBlock() -ArgumentList $Resource
   $null = Wait-Job -Job $j -Timeout $Timeout
 
   if ($j.State -eq 'Running') {
-    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)': {$ScriptBlock}"
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'"
     if ($PassThru) {
       $Resource
     }
+  } elseif ($j.State -eq 'Completed') {
+    Receive-Job -Job $j
   }
 }
diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index 582166525f6..dbd69f18dc7 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -90,6 +90,6 @@ foreach ($rg in $toDelete)
 Write-Host "Attempting to purge $($purgeableResources.Count) resources."
 $failedResources = @(Remove-PurgeableResources $purgeableResources -PassThru)
 if ($failedResources) {
-  Write-Warning "Failed to delete the following $($failedResources.Count) resources:"
+  Write-Warning "Timed out deleting the following $($failedResources.Count) resources. Please file an IcM ticket per resource type."
   $failedResources | Sort-Object AzsdkResourceType, AzsdkName | Format-Table -Property @{l='Type'; e={$_.AzsdkResourceType}}, @{l='Name'; e={$_.AzsdkName}}
 }

From c9766273ab14a43718a092dbbdc8ee78dc9e52f2 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 22:03:52 -0700
Subject: [PATCH 07/11] Add -ErrorAction to Receive-Job

Worked without terminating when run locally, but failed on the first error in the AzDO agent.
---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 8e7a4e701fb..5664654b3ab 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -183,6 +183,6 @@ function Wait-PurgeableResource {
       $Resource
     }
   } elseif ($j.State -eq 'Completed') {
-    Receive-Job -Job $j
+    Receive-Job -Job $j -ErrorAction Continue
   }
 }

From e24fefc901f416e21bad85ffadaa8a132a58ce17 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 12:38:20 -0700
Subject: [PATCH 08/11] Use $using:r instead of creating ScriptBlock

More idiomatic for passing ScriptBlocks to jobs.
---
 .../scripts/Helpers/Resource-Helpers.ps1      | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 5664654b3ab..8bc43a8c4ca 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -115,7 +115,8 @@ filter Remove-PurgeableResources {
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
+        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue }
       }
 
       'Managed HSM' {
@@ -124,15 +125,16 @@ filter Remove-PurgeableResources {
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
+        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
         Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
-          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
+          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($($using:r).Location)/deletedManagedHSMs/$($($using:r).Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
           if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-            Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+            Write-Warning "Successfully requested that Managed HSM '$($($using:r).Name)' be purged, but may take a few minutes before it is actually purged."
           } elseif ($response.Content) {
             $content = $response.Content | ConvertFrom-Json
             if ($content.error) {
               $err = $content.error
-              Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+              Write-Warning "Failed to deleted Managed HSM '$($($using:r).Name)': ($($err.code)) $($err.message)"
             }
           }
         }
@@ -166,15 +168,10 @@ function Wait-PurgeableResource {
     [switch] $PassThru
   )
 
-  # We build a new AST to pass the `$Resource` as `$r`, which is the `foreach` variable declared in `Remove-PurgeableResources` above.
-  # This is done to make writing a purge script feel natural without having to worry about scope since `[ScriptBlock].GetNewClosure()`
-  # does not capture `$r` appropriately. If the variable name in the `foreach` above is changed, it must be changed here as well.
-  $scriptBlockAst = [System.Management.Automation.Language.Parser]::ParseInput(@"
-  param (`$r)
-  $($ScriptBlock.Ast.EndBlock.ToString())
-"@, [ref] $null, [ref] $null)
-
-  $j = Start-ThreadJob -ScriptBlock $scriptBlockAst.GetScriptBlock() -ArgumentList $Resource
+  # ScriptBlocks need to use `$($using:r)` to capture the `$r` value in the `foreach` loop
+  # or they will not resolved when invoked as a job. See
+  # https://docs.microsoft.com/powershell/module/microsoft.powershell.core/about/about_remote_variables
+  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
   $null = Wait-Job -Job $j -Timeout $Timeout
 
   if ($j.State -eq 'Running') {

From dd769cb49a169e138ac836459acd906d87a72e41 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 16:39:48 -0700
Subject: [PATCH 09/11] Resolve PR feedback

---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 8bc43a8c4ca..ff61715545c 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -116,7 +116,9 @@ filter Remove-PurgeableResources {
         }
 
         # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue }
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
+          Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue
+        }
       }
 
       'Managed HSM' {

From 9e1a727bf00036f03cddb9b5d3dd8735dcbf60af Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 16:40:35 -0700
Subject: [PATCH 10/11] Change default DeleteAfterHours to 120

Resolves #1917
---
 eng/common/TestResources/New-TestResources.ps1    | 2 +-
 eng/common/TestResources/New-TestResources.ps1.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1
index 60f9f2bf2c0..7d5353202fe 100644
--- a/eng/common/TestResources/New-TestResources.ps1
+++ b/eng/common/TestResources/New-TestResources.ps1
@@ -50,7 +50,7 @@ param (
 
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
-    [int] $DeleteAfterHours = 48,
+    [int] $DeleteAfterHours = 120,
 
     [Parameter()]
     [string] $Location = '',
diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md
index 75c4676102e..b27dfba8c81 100644
--- a/eng/common/TestResources/New-TestResources.ps1.md
+++ b/eng/common/TestResources/New-TestResources.ps1.md
@@ -307,7 +307,7 @@ Aliases:
 
 Required: False
 Position: Named
-Default value: 48
+Default value: 120
 Accept pipeline input: False
 Accept wildcard characters: False
 ```

From c4eb0684da850134339a51a3831ea4ec903b85e7 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 21:20:11 -0700
Subject: [PATCH 11/11] Use the Az cmdlets built-in -AsJob

---
 .../scripts/Helpers/Resource-Helpers.ps1      | 70 +++++++++++--------
 1 file changed, 39 insertions(+), 31 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index ff61715545c..a73cff2f6fb 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -111,35 +111,35 @@ filter Remove-PurgeableResources {
     switch ($r.AzsdkResourceType) {
       'Key Vault' {
         if ($r.EnablePurgeProtection) {
-          # We will try anyway but will ignore errors
+          # We will try anyway but will ignore errors.
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
-          Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue
-        }
+        # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior.
+        Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob `
+          | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru
       }
 
       'Managed HSM' {
         if ($r.EnablePurgeProtection) {
-          # We will try anyway but will ignore errors
+          # We will try anyway but will ignore errors.
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
-          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($($using:r).Location)/deletedManagedHSMs/$($($using:r).Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
-          if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-            Write-Warning "Successfully requested that Managed HSM '$($($using:r).Name)' be purged, but may take a few minutes before it is actually purged."
-          } elseif ($response.Content) {
-            $content = $response.Content | ConvertFrom-Json
-            if ($content.error) {
-              $err = $content.error
-              Write-Warning "Failed to deleted Managed HSM '$($($using:r).Name)': ($($err.code)) $($err.message)"
-            }
-          }
-        }
+        # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured.
+        Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore -AsJob `
+          | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action {
+              param ( $response )
+              if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+                Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+              } elseif ($response.Content) {
+                $content = $response.Content | ConvertFrom-Json
+                if ($content.error) {
+                  $err = $content.error
+                  Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+                }
+              }
+            }.GetNewClosure()
       }
 
       default {
@@ -154,14 +154,20 @@ function Log($Message) {
   Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
 
-function Wait-PurgeableResource {
+function Wait-PurgeableResourceJob {
   param (
-    [Parameter(Mandatory=$true)]
-    [scriptblock] $ScriptBlock,
+    [Parameter(Mandatory=$true, ValueFromPipeline=$true)]
+    $Job,
 
+    # The resource is used for logging and to return if `-PassThru` is specified
+    # so we can easily see all resources that may be in a bad state when the script has completed.
     [Parameter(Mandatory=$true)]
     $Resource,
 
+    # Optional ScriptBlock should define params corresponding to the associated job's `Output` property.
+    [Parameter()]
+    [scriptblock] $Action,
+
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
     [int] $Timeout = 30,
@@ -170,18 +176,20 @@ function Wait-PurgeableResource {
     [switch] $PassThru
   )
 
-  # ScriptBlocks need to use `$($using:r)` to capture the `$r` value in the `foreach` loop
-  # or they will not resolved when invoked as a job. See
-  # https://docs.microsoft.com/powershell/module/microsoft.powershell.core/about/about_remote_variables
-  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
-  $null = Wait-Job -Job $j -Timeout $Timeout
+  $null = Wait-Job -Job $Job -Timeout $Timeout
+
+  if ($Job.State -eq 'Completed' -or $Job.State -eq 'Failed') {
+    $result = Receive-Job -Job $Job -ErrorAction Continue
+
+    if ($Action) {
+      $null = $Action.Invoke($result)
+    }
+  } else {
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job."
+    $Job.Cancel()
 
-  if ($j.State -eq 'Running') {
-    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'"
     if ($PassThru) {
       $Resource
     }
-  } elseif ($j.State -eq 'Completed') {
-    Receive-Job -Job $j -ErrorAction Continue
   }
 }