From 9335e30034774604fe2168cabaeab4a7b9759acb Mon Sep 17 00:00:00 2001
From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com>
Date: Wed, 3 Aug 2022 08:10:27 +0300
Subject: [PATCH] Run tre start/stop operations in parallel (#2394)

* run tre start/stop operations in parallel

* update changelog

* update changelog
---
 CHANGELOG.md                  |  1 +
 devops/scripts/control_tre.sh | 34 +++++++++++++++++++++-------------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f47f9adc2..8a62d005bd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ FEATURES:
 ENHANCEMENTS:
 
 * Guacamole logs are sent to Application Insights ([#2376](https://github.com/microsoft/AzureTRE/pull/2376))
+* `make tre-start/stop` run in parallel which saves ~5 minutes ([#2394](https://github.com/microsoft/AzureTRE/pull/2394))
 
 BUG FIXES:
 
diff --git a/devops/scripts/control_tre.sh b/devops/scripts/control_tre.sh
index f47299b511..0de6f371fa 100755
--- a/devops/scripts/control_tre.sh
+++ b/devops/scripts/control_tre.sh
@@ -21,7 +21,6 @@ if [[ $(az group list --output json --query "[?name=='${core_rg_name}'] | length
 fi
 
 az config set extension.use_dynamic_install=yes_without_prompt
-az extension add --name azure-firewall
 az --version
 
 if [[ "$1" == *"start"* ]]; then
@@ -29,7 +28,7 @@ if [[ "$1" == *"start"* ]]; then
     CURRENT_PUBLIC_IP=$(az network firewall ip-config list -f "${fw_name}" -g "${core_rg_name}" --query "[0].publicIpAddress" -o tsv)
     if [ -z "$CURRENT_PUBLIC_IP" ]; then
       echo "Starting Firewall - creating ip-config"
-      az network firewall ip-config create -f "${fw_name}" -g "${core_rg_name}" -n "fw-ip-configuration" --public-ip-address "pip-${fw_name}" --vnet-name "vnet-$TRE_ID" > /dev/null
+      az network firewall ip-config create -f "${fw_name}" -g "${core_rg_name}" -n "fw-ip-configuration" --public-ip-address "pip-${fw_name}" --vnet-name "vnet-$TRE_ID" > /dev/null &
     else
       echo "Firewall ip-config already exists"
     fi
@@ -37,7 +36,7 @@ if [[ "$1" == *"start"* ]]; then
 
   if [[ $(az network application-gateway list --output json --query "[?resourceGroup=='${core_rg_name}'&&name=='${agw_name}'&&operationalState=='Stopped'] | length(@)") != 0 ]]; then
     echo "Starting Application Gateway"
-    az network application-gateway start -g "${core_rg_name}" -n "${agw_name}"
+    az network application-gateway start -g "${core_rg_name}" -n "${agw_name}" &
   else
     echo "Application Gateway already running"
   fi
@@ -45,7 +44,7 @@ if [[ "$1" == *"start"* ]]; then
   az mysql server list --resource-group "${core_rg_name}" --query "[?userVisibleState=='Stopped'].name" -o tsv |
   while read -r mysql_name; do
     echo "Starting MySQL ${mysql_name}"
-    az mysql server start --resource-group "${core_rg_name}" --name "${mysql_name}"
+    az mysql server start --resource-group "${core_rg_name}" --name "${mysql_name}" &
   done
 
   az vmss list --resource-group "${core_rg_name}" --query "[].name" -o tsv |
@@ -53,14 +52,14 @@ if [[ "$1" == *"start"* ]]; then
     if [[ "$(az vmss list-instances --resource-group "${core_rg_name}" --name "${vmss_name}" --expand instanceView | \
       jq 'select(.[].instanceView.statuses[].code=="PowerState/deallocated") | length')" -gt 0 ]]; then
       echo "Starting VMSS ${vmss_name}"
-      az vmss start --resource-group "${core_rg_name}" --name "${vmss_name}"
+      az vmss start --resource-group "${core_rg_name}" --name "${vmss_name}" &
     fi
   done
 
   az vm list -d --resource-group "${core_rg_name}" --query "[?powerState!='VM running'].name" -o tsv |
   while read -r vm_name; do
     echo "Starting VM ${vm_name}"
-    az vm start --resource-group "${core_rg_name}" --name "${vm_name}"
+    az vm start --resource-group "${core_rg_name}" --name "${vm_name}" &
   done
 
   # We don't start workspace VMs despite maybe stopping them because we don't know if they need to be on.
@@ -71,7 +70,7 @@ elif [[ "$1" == *"stop"* ]]; then
 
     if [ -n "$IPCONFIG_NAME" ]; then
       echo "Deleting Firewall ip-config: $IPCONFIG_NAME"
-      az network firewall ip-config delete -f "${fw_name}" -n "$IPCONFIG_NAME" -g "${core_rg_name}"
+      az network firewall ip-config delete -f "${fw_name}" -n "$IPCONFIG_NAME" -g "${core_rg_name}" &
     else
       echo "No Firewall ip-config found"
     fi
@@ -79,7 +78,7 @@ elif [[ "$1" == *"stop"* ]]; then
 
   if [[ $(az network application-gateway list --output json --query "[?resourceGroup=='${core_rg_name}'&&name=='${agw_name}'&&operationalState=='Running'] | length(@)") != 0 ]]; then
     echo "Stopping Application Gateway"
-    az network application-gateway stop -g "${core_rg_name}" -n "${agw_name}"
+    az network application-gateway stop -g "${core_rg_name}" -n "${agw_name}" &
   else
     echo "Application Gateway already stopped"
   fi
@@ -87,19 +86,22 @@ elif [[ "$1" == *"stop"* ]]; then
   az mysql server list --resource-group "${core_rg_name}" --query "[?userVisibleState=='Ready'].name" -o tsv |
   while read -r mysql_name; do
     echo "Stopping MySQL ${mysql_name}"
-    az mysql server stop --resource-group "${core_rg_name}" --name "${mysql_name}"
+    az mysql server stop --resource-group "${core_rg_name}" --name "${mysql_name}" &
   done
 
   az vmss list --resource-group "${core_rg_name}" --query "[].name" -o tsv |
   while read -r vmss_name; do
-    echo "Deallocating VMSS ${vmss_name}"
-    az vmss deallocate --resource-group "${core_rg_name}" --name "${vmss_name}"
+    if [[ "$(az vmss list-instances --resource-group "${core_rg_name}" --name "${vmss_name}" --expand instanceView | \
+      jq 'select(.[].instanceView.statuses[].code=="PowerState/running") | length')" -gt 0 ]]; then
+      echo "Deallocating VMSS ${vmss_name}"
+      az vmss deallocate --resource-group "${core_rg_name}" --name "${vmss_name}" &
+    fi
   done
 
   az vm list -d --resource-group "${core_rg_name}" --query "[?powerState=='VM running'].name" -o tsv |
   while read -r vm_name; do
     echo "Deallocating VM ${vm_name}"
-    az vm deallocate --resource-group "${core_rg_name}" --name "${vm_name}"
+    az vm deallocate --resource-group "${core_rg_name}" --name "${vm_name}" &
   done
 
   # deallocating all VMs in workspaces
@@ -107,10 +109,16 @@ elif [[ "$1" == *"stop"* ]]; then
   az vm list --query "[?(starts_with(resourceGroup,'${core_rg_name}-ws') || starts_with(resourceGroup,'${core_rg_name^^}-WS')) && powerState=='VM running'][name, resourceGroup]" -o tsv |
   while read -r vm_name rg_name; do
     echo "Deallocating VM ${vm_name} in ${rg_name}"
-    az vm deallocate --resource-group "${rg_name}" --name "${vm_name}"
+    az vm deallocate --resource-group "${rg_name}" --name "${vm_name}" &
   done
 fi
 
+# for some reason the vm/vmss commands aren't considered as 'jobs', but this will still work in most cases
+# since firewall/appgw will take much longer to complete their change.
+echo "Waiting for all jobs to finish..."
+jobs
+wait
+
 # Report final FW status
 FW_STATE="Stopped"
 if [[ $(az network firewall list --output json --query "[?resourceGroup=='${core_rg_name}'&&name=='${fw_name}'] | length(@)") != 0 ]]; then