Merge branch 'main' into marrobi/issue2780

.github/workflows/deploy_tre.yml

@@ -32,6 +32,7 @@ jobs:
         ${{ (github.event_name == 'push' && 'extended or extended_aad')
         || 'extended or extended_aad or shared_services or airlock' }}
       environmentName: ${{ github.event.inputs.environment || 'CICD' }}
+      E2E_TESTS_NUMBER_PROCESSES: 1
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ secrets.ACR_NAME }}
@@ -60,4 +61,3 @@ jobs:
       CORE_APP_SERVICE_PLAN_SKU: ${{ secrets.CORE_APP_SERVICE_PLAN_SKU }}
       WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }}
       RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ secrets.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }}
-      E2E_TESTS_NUMBER_PROCESSES: "1"

.github/workflows/deploy_tre_branch.yml

@@ -58,6 +58,7 @@ jobs:
       prHeadSha: ${{ github.sha }}
       e2eTestsCustomSelector: ${{ github.event.inputs.e2eTestsCustomSelector }}
       environmentName: ${{ github.event.inputs.environment }}
+      E2E_TESTS_NUMBER_PROCESSES: 1
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.prepare-not-main.outputs.refid) }}
@@ -87,4 +88,3 @@ jobs:
       CORE_APP_SERVICE_PLAN_SKU: ${{ secrets.CORE_APP_SERVICE_PLAN_SKU }}
       WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }}
       RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ secrets.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }}
-      E2E_TESTS_NUMBER_PROCESSES: "1"

.github/workflows/deploy_tre_reusable.yml

@@ -27,6 +27,10 @@ on:  # yamllint disable-line rule:truthy
         description: The name of the Github Action's environment this will deploy into
         type: string
         required: true
+      E2E_TESTS_NUMBER_PROCESSES:
+        description: ""
+        type: number
+        required: false
     secrets:
       AAD_TENANT_ID:
         description: ""
@@ -112,9 +116,6 @@ on:  # yamllint disable-line rule:truthy
       RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE:
         description: ""
         required: false
-      E2E_TESTS_NUMBER_PROCESSES:
-        description: ""
-        required: false
 
 # This will prevent multiple runs of this entire workflow.
 # We should NOT cancel in progress runs as that can destabilize the environment.
@@ -787,7 +788,7 @@ jobs:
           TRE_ID: "${{ secrets.TRE_ID }}"
           IS_API_SECURED: false
           WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ secrets.WORKSPACE_APP_SERVICE_PLAN_SKU }}
-          E2E_TESTS_NUMBER_PROCESSES: ${{ secrets.E2E_TESTS_NUMBER_PROCESSES }}
+          E2E_TESTS_NUMBER_PROCESSES: ${{ inputs.E2E_TESTS_NUMBER_PROCESSES }}
 
       - name: Upload Test Results
         if: always()

.github/workflows/pr_comment_bot.yml

@@ -150,6 +150,7 @@ jobs:
         (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_services') ||
         (needs.pr_comment.outputs.command == 'run-tests' && '') }}
       environmentName: CICD
+      E2E_TESTS_NUMBER_PROCESSES: 1
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.pr_comment.outputs.prRefId) }}
@@ -176,4 +177,3 @@ jobs:
       TRE_ID: ${{ format('tre{0}', needs.pr_comment.outputs.prRefId) }}
       CI_CACHE_ACR_NAME: ${{ secrets.ACR_NAME }}
       TF_LOG: ${{ secrets.TF_LOG }}
-      E2E_TESTS_NUMBER_PROCESSES: "1"

CHANGELOG.md

@@ -1,7 +1,28 @@
 <!-- markdownlint-disable MD041 -->
 ## 0.9.0 (Unreleased)
+
 **BREAKING CHANGES & MIGRATIONS**:
 
+* Move to Azure **Firewall Policy** [#3107](https://github.com/microsoft/AzureTRE/pull/3107). This is a major version for the firewall shared service and will fail to automatically upgrade. You should follow these steps to complete it:
+  1. Let the system try to do the upgrade (via CI or `make tre-deploy`). It will fail but it's fine since now we have the new version published and registered.
+  2. Make a temporary network change with either of the following options:
+      * Azure Portal: find your TRE resource group and select the route table resource (named `rt-YOUR_TRE_ID`).
+        In the overview screen, find the `ResourceProcessorSubnet` (should be last in the subnet list), click on the `...` and select `Dissociate`.
+      * Azure CLI:
+        ```shell
+        az network vnet subnet update --resource-group rg-YOUR_TRE_ID --vnet-name vnet-YOUR_TRE_ID --name ResourceProcessorSubnet --remove routeTable
+        ```
+  4. Issue a patch API request to `force-update` the firewall to its new version.
+
+      One way to accomplish this is with the Swagger endpoint (/api/docs).
+      ![Force-update a service](./docs/assets/firewall-policy-migrate1.png)
+
+      If this endpoint is not on in your deployment - include `enable_swagger` in your `config.yaml` (see the sample file), or temporarly via the API resource on azure (named `api-YOUR_TRE-ID`) -> Configuration -> `ENABLE_SWAGGER` item.
+      ![Update API setting](./docs/assets/firewall-policy-migrate2.png)
+
+
+  :warning: Any custom rules you have added manually will be **lost** and you'll need to add it back after the upgrade has been completed.
+
 FEATURES:
 
 ENHANCEMENTS:
@@ -339,7 +360,7 @@ BUG FIXES:
 * API health check is also returned by accessing the root path at / ([#2469](https://github.com/microsoft/AzureTRE/pull/2469))
 * Temporary disable AppInsight's private endpoint in base workspace ([#2543](https://github.com/microsoft/AzureTRE/pull/2543))
 * Resource Processor execution optimization (`porter show`) for long-standing services ([#2542](https://github.com/microsoft/AzureTRE/pull/2542))
-* Move AML Compute deployment to use AzApi Terraform Provider {[#2555]((https://github.com/microsoft/AzureTRE/pull/2555))
+* Move AML Compute deployment to use AzApi Terraform Provider ([#2555](https://github.com/microsoft/AzureTRE/pull/2555))
 * Invalid token exceptions in the API app are caught, throwing 401 instead of 500 Internal server error ([#2572](https://github.com/microsoft/AzureTRE/pull/2572))
 
 COMPONENTS:

core/terraform/main.tf

@@ -11,11 +11,11 @@ terraform {
     }
     local = {
       source  = "hashicorp/local"
-      version = "~> 2.2.0"
+      version = "~> 2.3.0"
     }
     http = {
       source  = "hashicorp/http"
-      version = "~> 3.1.0"
+      version = "~> 3.2.0"
     }
   }
 

core/terraform/migrate.sh

@@ -158,6 +158,38 @@ if [ -n "${api_vnet_integration}" ]; then
     terraform apply -input=false -auto-approve ${PLAN_FILE}"
 fi
 
+# support changing the resource processor subnet size
+rp_subnet=$(echo "${terraform_show_json}" \
+  | jq -r 'select(.values.root_module.child_modules != null) .values.root_module.child_modules[] | select (.address=="module.network") | .resources[] | select(.address=="module.network.azurerm_subnet.resource_processor") | .values.id')
+if [ -n "${rp_subnet}" ]; then
+  set +o errexit
+  terraform plan -target "module.network.azurerm_subnet.resource_processor" -detailed-exitcode
+  plan_exit_code=$?
+  set -o errexit
+
+  if [ "${plan_exit_code}" == "2" ]; then
+    echo "Migrating ${rp_subnet}"
+    PLAN_FILE="tfplan$$"
+    TS=$(date +"%s")
+    LOG_FILE="${TS}-tre-core-migrate-rp-subnet.log"
+
+    # This variables are loaded in for us
+    # shellcheck disable=SC2154
+    "${terraform_wrapper_path}" \
+      -g "${TF_VAR_mgmt_resource_group_name}" \
+      -s "${TF_VAR_mgmt_storage_account_name}" \
+      -n "${TF_VAR_terraform_state_container_name}" \
+      -k "${TRE_ID}" \
+      -l "${LOG_FILE}" \
+      -c "terraform plan -destroy -target module.resource_processor_vmss_porter[0] \
+      -target azurerm_private_endpoint.sbpe \
+      -target azurerm_private_endpoint.mongo \
+      -out ${PLAN_FILE} && \
+      terraform apply -input=false -auto-approve ${PLAN_FILE}"
+  fi
+fi
+
+
 # this isn't a classic migration, but impacts how terraform handles the deployment in the next phase
 state_store_serverless=$(echo "${terraform_show_json}" \
   | jq 'select(.values.root_module.resources != null) | .values.root_module.resources[] | select(.address=="azurerm_cosmosdb_account.tre_db_account") | any(.values.capabilities[]; .name=="EnableServerless")')

core/terraform/network/locals.tf

@@ -1,22 +1,26 @@
 locals {
-  core_services_vnet_subnets = cidrsubnets(var.core_address_space, 4, 4, 4, 4, 2, 4, 4, 4, 4, 2)
-  # .1
+  core_services_vnet_subnets = cidrsubnets(var.core_address_space, 4, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4)
+  # Addresses examples are based on /22 CIDR
+  # .0
   firewall_subnet_address_space = local.core_services_vnet_subnets[0] # .0 - .63
   app_gw_subnet_address_prefix  = local.core_services_vnet_subnets[1] # .64 - .127
   bastion_subnet_address_prefix = local.core_services_vnet_subnets[2] # .128 - .191
   web_app_subnet_address_prefix = local.core_services_vnet_subnets[3] # .192 - .254
 
-  # .2
+  # .1
   shared_services_subnet_address_prefix = local.core_services_vnet_subnets[4] # .0 - .254
 
-  # replacing the aci
+  # .2
   airlock_processor_subnet_address_prefix     = local.core_services_vnet_subnets[5] # .0 - .63
   airlock_storage_subnet_address_prefix       = local.core_services_vnet_subnets[6] # .64 - .127
   airlock_events_subnet_address_prefix        = local.core_services_vnet_subnets[7] # .128 - .191
-  airlock_notifications_subnet_address_prefix = local.core_services_vnet_subnets[8] # .128 - .191
+  airlock_notifications_subnet_address_prefix = local.core_services_vnet_subnets[8] # .192 - .254
 
   # .3
-  resource_processor_subnet_address_prefix = local.core_services_vnet_subnets[9] # .0 - .254
+  resource_processor_subnet_address_prefix  = local.core_services_vnet_subnets[9]  # .0 - .63
+  firewall_management_subnet_address_prefix = local.core_services_vnet_subnets[10] # .64 - .127
+  # FREE = local.core_services_vnet_subnets[11] # .128 - .191
+  # FREE = local.core_services_vnet_subnets[12] # .192 - .254
 
   tre_core_tags = {
     tre_id              = var.tre_id

core/terraform/network/network.tf

@@ -130,14 +130,22 @@ resource "azurerm_subnet" "airlock_events" {
   address_prefixes     = [local.airlock_events_subnet_address_prefix]
   # notice that private endpoints do not adhere to NSG rules
   private_endpoint_network_policies_enabled = false
-  depends_on                                = [azurerm_subnet.airlock_events]
+  depends_on                                = [azurerm_subnet.airlock_storage]
 
   # Eventgrid CAN'T send messages over private endpoints, hence we need to allow service endpoints to the service bus
   # We are using service endpoints + managed identity to send these messaages
   # https://docs.microsoft.com/en-us/azure/event-grid/consume-private-endpoints
   service_endpoints = ["Microsoft.ServiceBus"]
 }
 
+resource "azurerm_subnet" "firewall_management" {
+  name                 = "AzureFirewallManagementSubnet"
+  virtual_network_name = azurerm_virtual_network.core.name
+  resource_group_name  = var.resource_group_name
+  address_prefixes     = [local.firewall_management_subnet_address_prefix]
+  depends_on           = [azurerm_subnet.airlock_events]
+}
+
 resource "azurerm_ip_group" "resource_processor" {
   name                = "ipg-resource-processor"
   location            = var.location
@@ -146,3 +154,21 @@ resource "azurerm_ip_group" "resource_processor" {
   tags                = local.tre_core_tags
   lifecycle { ignore_changes = [tags] }
 }
+
+resource "azurerm_ip_group" "shared" {
+  name                = "ipg-shared"
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  cidrs               = [local.shared_services_subnet_address_prefix]
+  tags                = local.tre_core_tags
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_ip_group" "webapp" {
+  name                = "ipg-web-app"
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  cidrs               = [local.web_app_subnet_address_prefix]
+  tags                = local.tre_core_tags
+  lifecycle { ignore_changes = [tags] }
+}

core/terraform/network/network_security_groups.tf

@@ -107,7 +107,7 @@ resource "azurerm_subnet_network_security_group_association" "bastion" {
   subnet_id                 = azurerm_subnet.bastion.id
   network_security_group_id = azurerm_network_security_group.bastion.id
   # depend on the last subnet we created in the vnet
-  depends_on = [azurerm_subnet.airlock_events]
+  depends_on = [azurerm_subnet.firewall_management]
 }
 
 # Network security group for Application Gateway

core/terraform/notebooks.tf

@@ -0,0 +1,15 @@
+data "http" "firewall_workbook_json" {
+  url = "https://raw.githubusercontent.com/Azure/Azure-Network-Security/master/Azure%20Firewall/Workbook%20-%20Azure%20Firewall%20Monitor%20Workbook/Azure%20Firewall_Gallery.json"
+}
+
+resource "random_uuid" "firewall_workbook" {
+}
+
+resource "azurerm_application_insights_workbook" "firewall" {
+  name                = random_uuid.firewall_workbook.result
+  location            = azurerm_resource_group.core.location
+  resource_group_name = azurerm_resource_group.core.name
+  display_name        = "Azure Firewall Workbook ${var.tre_id}"
+  data_json           = data.http.firewall_workbook_json.response_body
+  tags                = local.tre_core_tags
+}

core/terraform/variables.tf

@@ -20,6 +20,10 @@ variable "acr_name" {
 variable "core_address_space" {
   type        = string
   description = "Core services VNET Address Space"
+  validation {
+    condition     = parseint(element(split("/", var.core_address_space), 1), 10) > 0 && parseint(element(split("/", var.core_address_space), 1), 10) <= 22
+    error_message = "core_address_space size should be /22 or larger"
+  }
 }
 
 variable "tre_address_space" {

core/version.txt

@@ -1 +1 @@
-__version__ = "0.5.3"
+__version__ = "0.6.0"

devops/scripts/control_tre.sh

@@ -66,13 +66,14 @@ if [[ "$1" == *"start"* ]]; then
 
 elif [[ "$1" == *"stop"* ]]; then
   if [[ $(az network firewall list --output json --query "[?resourceGroup=='${core_rg_name}'&&name=='${fw_name}'] | length(@)") != 0 ]]; then
+    fw_sku=$(az network firewall show -n "${fw_name}" -g "${core_rg_name}" --query "sku.tier" -o tsv)
     IPCONFIG_NAME=$(az network firewall ip-config list -f "${fw_name}" -g "${core_rg_name}" --query "[0].name" -o tsv)
 
-    if [ -n "$IPCONFIG_NAME" ]; then
+    if [ -n "$IPCONFIG_NAME" ] && [ "${fw_sku}" != "Basic" ]; then
       echo "Deleting Firewall ip-config: $IPCONFIG_NAME"
       az network firewall ip-config delete -f "${fw_name}" -n "$IPCONFIG_NAME" -g "${core_rg_name}" &
     else
-      echo "No Firewall ip-config found"
+      echo "No Firewall ip-config found or SKU (${fw_sku}) doesn't allow deallocation"
     fi
   fi
 

templates/shared_services/firewall/parameters.json

@@ -45,6 +45,12 @@
       "source": {
         "env": "NETWORK_RULE_COLLECTIONS"
       }
+    },
+    {
+      "name": "sku_tier",
+      "source": {
+        "env": "SKU_TIER"
+      }
     }
   ]
 }