From e72d3cbda8272230aed5ddad27c19a322559eac8 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 14:49:26 -0700
Subject: [PATCH 01/10] Attempt to purge all vaults, managed HSMs

Reverts #1910. Vaults and managed HSMs are automatically purged on their purge date. The point was to purge them daily to preserve capacity. The default purge date is +90 days.
---
 .../scripts/Helpers/Resource-Helpers.ps1      | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 188639d46d4a6..97ecf1c122298 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -7,6 +7,18 @@ function Get-PurgeableGroupResources {
   )
   $purgeableResources = @()
 
+  # Discover Managed HSMs first since they are a premium resource.
+  Write-Verbose "Retrieving deleted Managed HSMs from resource group $ResourceGroupName"
+
+  # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
+  $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
+
+  if ($deletedHsms) {
+    Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
+    $purgeableResources += $deletedHsms
+  }
+
   Write-Verbose "Retrieving deleted Key Vaults from resource group $ResourceGroupName"
 
   # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled.
@@ -21,34 +33,13 @@ function Get-PurgeableGroupResources {
     $purgeableResources += $deletedKeyVaults
   }
 
-  Write-Verbose "Retrieving deleted Managed HSMs from resource group $ResourceGroupName"
-
-  # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
-  $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
-
-  if ($deletedHsms) {
-    Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
-    $purgeableResources += $deletedHsms
-  }
-
   return $purgeableResources
 }
 function Get-PurgeableResources {
   $purgeableResources = @()
   $subscriptionId = (Get-AzContext).Subscription.Id
 
-  Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
-
-  # Get deleted Key Vaults for the current subscription.
-  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
-
-  if ($deletedKeyVaults) {
-    Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
-    $purgeableResources += $deletedKeyVaults
-  }
-
+  # Discover Managed HSMs first since they are a premium resource.
   Write-Verbose "Retrieving deleted Managed HSMs from subscription $subscriptionId"
 
   # Get deleted Managed HSMs for the current subscription.
@@ -75,6 +66,17 @@ function Get-PurgeableResources {
     }
   }
 
+  Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
+
+  # Get deleted Key Vaults for the current subscription.
+  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+
+  if ($deletedKeyVaults) {
+    Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
+    $purgeableResources += $deletedKeyVaults
+  }
+
   return $purgeableResources
 }
 

From 50a85278a35af26f0f41c324d40c976a14babf62 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 17:39:41 -0700
Subject: [PATCH 02/10] Add timeout and more logging

---
 .../scripts/Helpers/Resource-Helpers.ps1      | 74 ++++++++++++++-----
 1 file changed, 57 insertions(+), 17 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 97ecf1c122298..b3dba47e0012b 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -12,7 +12,8 @@ function Get-PurgeableGroupResources {
 
   # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
   $deletedHsms = Get-AzKeyVaultManagedHsm -ResourceGroupName $ResourceGroupName -ErrorAction Ignore `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru `
+    | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
 
   if ($deletedHsms) {
     Write-Verbose "Found $($deletedHsms.Count) deleted Managed HSMs to potentially purge."
@@ -25,8 +26,9 @@ function Get-PurgeableGroupResources {
   $deletedKeyVaults = Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object {
     # Enumerating vaults from a resource group does not return all properties we required.
     Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } `
-      | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
-  }
+      | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
+      | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
+    }
 
   if ($deletedKeyVaults) {
     Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
@@ -35,6 +37,7 @@ function Get-PurgeableGroupResources {
 
   return $purgeableResources
 }
+
 function Get-PurgeableResources {
   $purgeableResources = @()
   $subscriptionId = (Get-AzContext).Subscription.Id
@@ -51,6 +54,7 @@ function Get-PurgeableResources {
     foreach ($r in $content.value) {
       $deletedHsms += [pscustomobject] @{
         AzsdkResourceType = 'Managed HSM'
+        AzsdkName = $r.name
         Id = $r.id
         Name = $r.name
         Location = $r.properties.location
@@ -70,7 +74,8 @@ function Get-PurgeableResources {
 
   # Get deleted Key Vaults for the current subscription.
   $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
-    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+    | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
+    | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
 
   if ($deletedKeyVaults) {
     Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge."
@@ -85,7 +90,14 @@ function Get-PurgeableResources {
 filter Remove-PurgeableResources {
   param (
     [Parameter(Position=0, ValueFromPipeline=$true)]
-    [object[]] $Resource
+    [object[]] $Resource,
+
+    [Parameter()]
+    [ValidateRange(1, [int]::MaxValue)]
+    [int] $Timeout = 30,
+
+    [Parameter()]
+    [switch] $PassThru
   )
 
   if (!$Resource) {
@@ -95,38 +107,39 @@ filter Remove-PurgeableResources {
   $subscriptionId = (Get-AzContext).Subscription.Id
 
   foreach ($r in $Resource) {
+    Log "Attempting to purge $($r.AzsdkResourceType) '$($r.AzsdkName)'"
     switch ($r.AzsdkResourceType) {
       'Key Vault' {
-        Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'"
         if ($r.EnablePurgeProtection) {
           # We will try anyway but will ignore errors
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue
+        Wait-PurgeableResource { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue } -Timeout:$Timeout -PassThru:$PassThru
       }
 
       'Managed HSM' {
-        Log "Attempting to purge $($r.AzsdkResourceType) '$($r.Name)'"
         if ($r.EnablePurgeProtection) {
           # We will try anyway but will ignore errors
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
-        if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-          Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
-        } elseif ($response.Content) {
-          $content = $response.Content | ConvertFrom-Json
-          if ($content.error) {
-            $err = $content.error
-            Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+        Wait-PurgeableResource -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
+          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
+          if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+            Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+          } elseif ($response.Content) {
+            $content = $response.Content | ConvertFrom-Json
+            if ($content.error) {
+              $err = $content.error
+              Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+            }
           }
         }
       }
 
       default {
-        Write-Warning "Cannot purge resource type $($r.AzsdkResourceType). Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
+        Write-Warning "Cannot purge $($r.AzsdkResourceType) '$($r.AzsdkName)'. Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
       }
     }
   }
@@ -136,3 +149,30 @@ filter Remove-PurgeableResources {
 function Log($Message) {
   Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
+
+function Wait-PurgeableResource {
+  param (
+    [Parameter(Mandatory=$true, Position=0)]
+    [scriptblock] $ScriptBlock,
+
+    [Parameter(Mandatory=$true, Position=1)]
+    [object] $Resource,
+
+    [Parameter()]
+    [ValidateRange(1, [int]::MaxValue)]
+    [int] $Timeout = 30,
+
+    [Parameter()]
+    [switch] $PassThru
+  )
+
+  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
+  $null = Wait-Job -Job $j -Timeout $Timeout
+
+  if ($j.State -eq 'Running') {
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$(Resource.AzsdkName)': {$ScriptBlock}"
+    if ($PassThru) {
+      $Resource
+    }
+  }
+}

From 40a1c366074259c8f68bdc0cafaf6b60a6da8707 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 17:53:09 -0700
Subject: [PATCH 03/10] Pass required -Resource

---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index b3dba47e0012b..dbffde230ae1a 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -115,7 +115,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue } -Timeout:$Timeout -PassThru:$PassThru
+        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
       }
 
       'Managed HSM' {
@@ -124,7 +124,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
+        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
           $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
           if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
             Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."

From bfe6c350a8d15d548e31333237b1f84646e91b46 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 18:31:59 -0700
Subject: [PATCH 04/10] Fix log message

---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index dbffde230ae1a..e064f1b683143 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -170,7 +170,7 @@ function Wait-PurgeableResource {
   $null = Wait-Job -Job $j -Timeout $Timeout
 
   if ($j.State -eq 'Running') {
-    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$(Resource.AzsdkName)': {$ScriptBlock}"
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)': {$ScriptBlock}"
     if ($PassThru) {
       $Resource
     }

From 5719811e9f8e7f8eb39b4345a634eb8ef35dcb8b Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 21:26:45 -0700
Subject: [PATCH 05/10] Ensure the $Resource is correctly captured

Added comment to new code explaining why, since ScriptBlock.GetNewClosure() is not working as expected.
---
 .../scripts/Helpers/Resource-Helpers.ps1      | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index e064f1b683143..8e7a4e701fb96 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -73,7 +73,7 @@ function Get-PurgeableResources {
   Write-Verbose "Retrieving deleted Key Vaults from subscription $subscriptionId"
 
   # Get deleted Key Vaults for the current subscription.
-  $deletedKeyVaults  = Get-AzKeyVault -InRemovedState `
+  $deletedKeyVaults = Get-AzKeyVault -InRemovedState `
     | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru `
     | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru
 
@@ -115,7 +115,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
       }
 
       'Managed HSM' {
@@ -124,7 +124,7 @@ filter Remove-PurgeableResources {
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Resource $r -Timeout:$Timeout -PassThru:$PassThru -ScriptBlock {
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
           $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
           if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
             Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
@@ -152,11 +152,11 @@ function Log($Message) {
 
 function Wait-PurgeableResource {
   param (
-    [Parameter(Mandatory=$true, Position=0)]
+    [Parameter(Mandatory=$true)]
     [scriptblock] $ScriptBlock,
 
-    [Parameter(Mandatory=$true, Position=1)]
-    [object] $Resource,
+    [Parameter(Mandatory=$true)]
+    $Resource,
 
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
@@ -166,13 +166,23 @@ function Wait-PurgeableResource {
     [switch] $PassThru
   )
 
-  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
+  # We build a new AST to pass the `$Resource` as `$r`, which is the `foreach` variable declared in `Remove-PurgeableResources` above.
+  # This is done to make writing a purge script feel natural without having to worry about scope since `[ScriptBlock].GetNewClosure()`
+  # does not capture `$r` appropriately. If the variable name in the `foreach` above is changed, it must be changed here as well.
+  $scriptBlockAst = [System.Management.Automation.Language.Parser]::ParseInput(@"
+  param (`$r)
+  $($ScriptBlock.Ast.EndBlock.ToString())
+"@, [ref] $null, [ref] $null)
+
+  $j = Start-ThreadJob -ScriptBlock $scriptBlockAst.GetScriptBlock() -ArgumentList $Resource
   $null = Wait-Job -Job $j -Timeout $Timeout
 
   if ($j.State -eq 'Running') {
-    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)': {$ScriptBlock}"
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'"
     if ($PassThru) {
       $Resource
     }
+  } elseif ($j.State -eq 'Completed') {
+    Receive-Job -Job $j
   }
 }

From 934c0837992211e74eff1f4a4dce0ff0c1f9b26b Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Mon, 16 Aug 2021 22:03:52 -0700
Subject: [PATCH 06/10] Add -ErrorAction to Receive-Job

Worked without terminating when run locally, but failed on the first error in the AzDO agent.
---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 8e7a4e701fb96..5664654b3ab2d 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -183,6 +183,6 @@ function Wait-PurgeableResource {
       $Resource
     }
   } elseif ($j.State -eq 'Completed') {
-    Receive-Job -Job $j
+    Receive-Job -Job $j -ErrorAction Continue
   }
 }

From bfa5c06fec1d0c5e918131de59d8b6ecbe8dfcf4 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 12:38:20 -0700
Subject: [PATCH 07/10] Use $using:r instead of creating ScriptBlock

More idiomatic for passing ScriptBlocks to jobs.
---
 .../scripts/Helpers/Resource-Helpers.ps1      | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 5664654b3ab2d..8bc43a8c4cafb 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -115,7 +115,8 @@ filter Remove-PurgeableResources {
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue }
+        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue }
       }
 
       'Managed HSM' {
@@ -124,15 +125,16 @@ filter Remove-PurgeableResources {
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
+        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
         Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
-          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
+          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($($using:r).Location)/deletedManagedHSMs/$($($using:r).Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
           if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-            Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+            Write-Warning "Successfully requested that Managed HSM '$($($using:r).Name)' be purged, but may take a few minutes before it is actually purged."
           } elseif ($response.Content) {
             $content = $response.Content | ConvertFrom-Json
             if ($content.error) {
               $err = $content.error
-              Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+              Write-Warning "Failed to deleted Managed HSM '$($($using:r).Name)': ($($err.code)) $($err.message)"
             }
           }
         }
@@ -166,15 +168,10 @@ function Wait-PurgeableResource {
     [switch] $PassThru
   )
 
-  # We build a new AST to pass the `$Resource` as `$r`, which is the `foreach` variable declared in `Remove-PurgeableResources` above.
-  # This is done to make writing a purge script feel natural without having to worry about scope since `[ScriptBlock].GetNewClosure()`
-  # does not capture `$r` appropriately. If the variable name in the `foreach` above is changed, it must be changed here as well.
-  $scriptBlockAst = [System.Management.Automation.Language.Parser]::ParseInput(@"
-  param (`$r)
-  $($ScriptBlock.Ast.EndBlock.ToString())
-"@, [ref] $null, [ref] $null)
-
-  $j = Start-ThreadJob -ScriptBlock $scriptBlockAst.GetScriptBlock() -ArgumentList $Resource
+  # ScriptBlocks need to use `$($using:r)` to capture the `$r` value in the `foreach` loop
+  # or they will not resolved when invoked as a job. See
+  # https://docs.microsoft.com/powershell/module/microsoft.powershell.core/about/about_remote_variables
+  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
   $null = Wait-Job -Job $j -Timeout $Timeout
 
   if ($j.State -eq 'Running') {

From 930ce57a56491a5b492ac218e3624f5ea28b2e09 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 16:39:48 -0700
Subject: [PATCH 08/10] Resolve PR feedback

---
 eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index 8bc43a8c4cafb..ff61715545c8c 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -116,7 +116,9 @@ filter Remove-PurgeableResources {
         }
 
         # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock { Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue }
+        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
+          Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue
+        }
       }
 
       'Managed HSM' {

From 85116bc3c3f21db6d8327fe763be939f33081f2f Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 16:40:35 -0700
Subject: [PATCH 09/10] Change default DeleteAfterHours to 120

Resolves #1917
---
 eng/common/TestResources/New-TestResources.ps1    | 2 +-
 eng/common/TestResources/New-TestResources.ps1.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1
index 60f9f2bf2c066..7d5353202fea6 100644
--- a/eng/common/TestResources/New-TestResources.ps1
+++ b/eng/common/TestResources/New-TestResources.ps1
@@ -50,7 +50,7 @@ param (
 
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
-    [int] $DeleteAfterHours = 48,
+    [int] $DeleteAfterHours = 120,
 
     [Parameter()]
     [string] $Location = '',
diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md
index 75c4676102e75..b27dfba8c81a7 100644
--- a/eng/common/TestResources/New-TestResources.ps1.md
+++ b/eng/common/TestResources/New-TestResources.ps1.md
@@ -307,7 +307,7 @@ Aliases:
 
 Required: False
 Position: Named
-Default value: 48
+Default value: 120
 Accept pipeline input: False
 Accept wildcard characters: False
 ```

From 547c501a706337d834f42586a8281d0e0cf9d247 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@microsoft.com>
Date: Tue, 17 Aug 2021 21:20:11 -0700
Subject: [PATCH 10/10] Use the Az cmdlets built-in -AsJob

---
 .../scripts/Helpers/Resource-Helpers.ps1      | 70 +++++++++++--------
 1 file changed, 39 insertions(+), 31 deletions(-)

diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
index ff61715545c8c..a73cff2f6fb3b 100644
--- a/eng/common/scripts/Helpers/Resource-Helpers.ps1
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -111,35 +111,35 @@ filter Remove-PurgeableResources {
     switch ($r.AzsdkResourceType) {
       'Key Vault' {
         if ($r.EnablePurgeProtection) {
-          # We will try anyway but will ignore errors
+          # We will try anyway but will ignore errors.
           Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
-          Remove-AzKeyVault -VaultName $($using:r).VaultName -Location $($using:r).Location -InRemovedState -Force -ErrorAction Continue
-        }
+        # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior.
+        Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob `
+          | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru
       }
 
       'Managed HSM' {
         if ($r.EnablePurgeProtection) {
-          # We will try anyway but will ignore errors
+          # We will try anyway but will ignore errors.
           Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
         }
 
-        # Using `$($using:r)` in the `-ScriptBlock` to make sure `$r` is captured for jobs.
-        Wait-PurgeableResource -Resource $r -Timeout $Timeout -PassThru:$PassThru -ScriptBlock {
-          $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($($using:r).Location)/deletedManagedHSMs/$($($using:r).Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
-          if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
-            Write-Warning "Successfully requested that Managed HSM '$($($using:r).Name)' be purged, but may take a few minutes before it is actually purged."
-          } elseif ($response.Content) {
-            $content = $response.Content | ConvertFrom-Json
-            if ($content.error) {
-              $err = $content.error
-              Write-Warning "Failed to deleted Managed HSM '$($($using:r).Name)': ($($err.code)) $($err.message)"
-            }
-          }
-        }
+        # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured.
+        Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore -AsJob `
+          | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action {
+              param ( $response )
+              if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+                Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+              } elseif ($response.Content) {
+                $content = $response.Content | ConvertFrom-Json
+                if ($content.error) {
+                  $err = $content.error
+                  Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+                }
+              }
+            }.GetNewClosure()
       }
 
       default {
@@ -154,14 +154,20 @@ function Log($Message) {
   Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
 
-function Wait-PurgeableResource {
+function Wait-PurgeableResourceJob {
   param (
-    [Parameter(Mandatory=$true)]
-    [scriptblock] $ScriptBlock,
+    [Parameter(Mandatory=$true, ValueFromPipeline=$true)]
+    $Job,
 
+    # The resource is used for logging and to return if `-PassThru` is specified
+    # so we can easily see all resources that may be in a bad state when the script has completed.
     [Parameter(Mandatory=$true)]
     $Resource,
 
+    # Optional ScriptBlock should define params corresponding to the associated job's `Output` property.
+    [Parameter()]
+    [scriptblock] $Action,
+
     [Parameter()]
     [ValidateRange(1, [int]::MaxValue)]
     [int] $Timeout = 30,
@@ -170,18 +176,20 @@ function Wait-PurgeableResource {
     [switch] $PassThru
   )
 
-  # ScriptBlocks need to use `$($using:r)` to capture the `$r` value in the `foreach` loop
-  # or they will not resolved when invoked as a job. See
-  # https://docs.microsoft.com/powershell/module/microsoft.powershell.core/about/about_remote_variables
-  $j = Start-ThreadJob -ScriptBlock $ScriptBlock
-  $null = Wait-Job -Job $j -Timeout $Timeout
+  $null = Wait-Job -Job $Job -Timeout $Timeout
+
+  if ($Job.State -eq 'Completed' -or $Job.State -eq 'Failed') {
+    $result = Receive-Job -Job $Job -ErrorAction Continue
+
+    if ($Action) {
+      $null = $Action.Invoke($result)
+    }
+  } else {
+    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job."
+    $Job.Cancel()
 
-  if ($j.State -eq 'Running') {
-    Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'"
     if ($PassThru) {
       $Resource
     }
-  } elseif ($j.State -eq 'Completed') {
-    Receive-Job -Job $j -ErrorAction Continue
   }
 }