Refactor Azure Monitor in core

devops/scripts/destroy_env_no_terraform.sh

@@ -95,23 +95,10 @@ echo "Looking for diagnostic settings..."
 # using xargs to run in parallel.
 az resource list --resource-group "${core_tre_rg}" --query '[].[id]' -o tsv | xargs -P 10 -I {} bash -c 'delete_resource_diagnostic "{}"'
 
+tre_id=${core_tre_rg#"rg-"}
 
 # purge keyvault if possible (makes it possible to reuse the same tre_id later)
 # this has to be done before we delete the resource group since we might not wait for it to complete
-
-# DEBUG START
-# This section is to aid debugging an issue where keyvaults aren't being deleted and purged
-echo "keyvault properties:"
-az keyvault list --resource-group "${core_tre_rg}" --query "[].properties"
-echo "keyvault purge protection evaluation result:"
-az keyvault list --resource-group "${core_tre_rg}" --query "[?properties.enablePurgeProtection==``null``] | length (@)"
-
-if [[ -n ${SHOW_KEYVAULT_DEBUG_ON_DESTROY:-} ]]; then
-  az keyvault list --resource-group "${core_tre_rg}" --query "[].properties" --debug
-fi
-# DEBUG END
-
-tre_id=${core_tre_rg#"rg-"}
 keyvault_name="kv-${tre_id}"
 keyvault=$(az keyvault show --name "${keyvault_name}" --resource-group "${core_tre_rg}" || echo 0)
 if [ "${keyvault}" != "0" ]; then
@@ -150,6 +137,14 @@ else
   echo "Resource group ${core_tre_rg} doesn't have a keyvault without purge protection."
 fi
 
+# linked storage accounts don't get deleted with the workspace
+workspace_name="log-${tre_id}"
+workspace=$(az monitor log-analytics workspace show --workspace-name "${workspace_name}" --resource-group "${core_tre_rg}" || echo 0)
+if [ "${workspace}" != "0" ]; then
+  az monitor log-analytics workspace linked-storage list -g "${core_tre_rg}" --workspace-name "${workspace_name}" -o tsv --query '[].id' \
+  | xargs -P 10 -I {} az rest --method delete --uri "{}?api-version=2020-08-01"
+fi
+
 # this will find the mgmt, core resource groups as well as any workspace ones
 # we are reverse-sorting to first delete the workspace groups (might not be
 # good enough because we use no-wait sometimes)

templates/core/terraform/azure-monitor/app_insights_byo_storage.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "app_insights_name": {
+      "type": "String"
+    },
+    "storage_account_resource_id": {
+      "type": "String"
+    }
+  },
+  "variables": {},
+  "resources": [
+    {
+      "name": "[concat(parameters('app_insights_name'), '/serviceprofiler')]",
+      "type": "microsoft.insights/components/linkedStorageAccounts",
+      "apiVersion": "2020-03-01-preview",
+      "properties": {
+        "linkedStorageAccount": "[parameters('storage_account_resource_id')]"
+      }
+    }
+  ]
+}

templates/core/terraform/azure-monitor/azure-monitor.tf

@@ -1,108 +1,131 @@
+# Log Analytics
 resource "azurerm_log_analytics_workspace" "core" {
-  name                = "log-${var.tre_id}"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  retention_in_days   = 30
-  sku                 = "PerGB2018"
-  tags                = local.tre_core_tags
+  name                       = "log-${var.tre_id}"
+  resource_group_name        = var.resource_group_name
+  location                   = var.location
+  retention_in_days          = 30
+  sku                        = "PerGB2018"
+  tags                       = var.tre_core_tags
+  internet_ingestion_enabled = false
 
   lifecycle { ignore_changes = [tags] }
 }
 
-# Storage account for Application Insights
+# Storage account for Azure Monitor ingestion
 # Because Private Link is enabled on Application Performance Management (APM), Bring Your Own Storage (BYOS) approach is required
-resource "azurerm_storage_account" "app_insights" {
-  name                            = lower(replace("stappinsights${var.tre_id}", "-", ""))
+resource "azurerm_storage_account" "az_monitor" {
+  name                            = lower(replace("stazmonitor${var.tre_id}", "-", ""))
   resource_group_name             = var.resource_group_name
   location                        = var.location
   account_kind                    = "StorageV2"
   account_tier                    = "Standard"
   account_replication_type        = "LRS"
   allow_nested_items_to_be_public = false
-  tags                            = local.tre_core_tags
+  tags                            = var.tre_core_tags
 
   lifecycle { ignore_changes = [tags] }
 }
 
-data "local_file" "app_insights_arm_template" {
-  filename = "${path.module}/app_insights.json"
+resource "azurerm_log_analytics_linked_storage_account" "workspace_storage_ingestion" {
+  data_source_type      = "ingestion"
+  resource_group_name   = var.resource_group_name
+  workspace_resource_id = azurerm_log_analytics_workspace.core.id
+  storage_account_ids   = [azurerm_storage_account.az_monitor.id]
 }
 
-# Application Insights
-# Deployed using ARM template, because Terraform's azurerm_application_insights does not support linked storage account
-resource "azurerm_resource_group_template_deployment" "app_insights_core" {
-  name                = local.app_insights_name
+resource "azurerm_log_analytics_linked_storage_account" "workspace_storage_customlogs" {
+  data_source_type      = "customlogs"
+  resource_group_name   = var.resource_group_name
+  workspace_resource_id = azurerm_log_analytics_workspace.core.id
+  storage_account_ids   = [azurerm_storage_account.az_monitor.id]
+}
+
+resource "azurerm_monitor_private_link_scope" "ampls_core" {
+  name                = "ampls-${var.tre_id}"
   resource_group_name = var.resource_group_name
-  deployment_mode     = "Incremental"
-  template_content    = data.local_file.app_insights_arm_template.content
+  tags                = var.tre_core_tags
 
-  parameters_content = jsonencode({
-    "app_insights_name" = {
-      value = local.app_insights_name
-    }
-    "location" = {
-      value = var.location
-    }
-    "log_analytics_workspace_id" = {
-      value = azurerm_log_analytics_workspace.core.id
-    }
-    "application_type" = {
-      value = "web"
-    }
-    "storage_account_name" = {
-      value = azurerm_storage_account.app_insights.name
-    }
-    "tre_core_tags" = {
-      value = local.tre_core_tags
-    }
-  })
+  lifecycle { ignore_changes = [tags] }
 }
 
-data "local_file" "ampls_arm_template" {
-  filename = "${path.module}/ampls.json"
+resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" {
+  name                = "ampls-log-anaytics-service"
+  resource_group_name = var.resource_group_name
+  scope_name          = azurerm_monitor_private_link_scope.ampls_core.name
+  linked_resource_id  = azurerm_log_analytics_workspace.core.id
 }
 
-# Azure Monitor Private Link Scope
-# See https://docs.microsoft.com/azure/azure-monitor/logs/private-link-security
-resource "azurerm_resource_group_template_deployment" "ampls_core" {
-  name                = "ampls-${var.tre_id}"
+
+
+# Application Insights
+
+resource "azurerm_application_insights" "core" {
+  name                                = "appi-${var.tre_id}"
+  location                            = var.location
+  resource_group_name                 = var.resource_group_name
+  workspace_id                        = azurerm_log_analytics_workspace.core.id
+  application_type                    = "web"
+  internet_ingestion_enabled          = false
+  force_customer_storage_for_profiler = true
+  tags                                = var.tre_core_tags
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" {
+  name                = "ampls-app-insights-service"
   resource_group_name = var.resource_group_name
-  deployment_mode     = "Incremental"
-  template_content    = data.local_file.ampls_arm_template.content
+  scope_name          = azurerm_monitor_private_link_scope.ampls_core.name
+  linked_resource_id  = azurerm_application_insights.core.id
+}
+
+data "local_file" "app_insights_byo_storage_arm_template" {
+  filename = "${path.module}/app_insights_byo_storage.json"
+}
 
+# Deployed using ARM template, because Terraform's azurerm_application_insights does not support linked storage account
+# https://docs.microsoft.com/en-us/azure/azure-monitor/app/profiler-bring-your-own-storage
+resource "azurerm_resource_group_template_deployment" "app_insights_byo_storage" {
+  name                = azurerm_application_insights.core.name
+  resource_group_name = var.resource_group_name
+  deployment_mode     = "Incremental"
+  template_content    = data.local_file.app_insights_byo_storage_arm_template.content
 
   parameters_content = jsonencode({
-    "private_link_scope_name" = {
-      value = "ampls-${var.tre_id}"
-    }
-    "workspace_name" = {
-      value = azurerm_log_analytics_workspace.core.name
-    }
     "app_insights_name" = {
-      value = local.app_insights_name
+      value = azurerm_application_insights.core.name
     }
-    "tre_core_tags" = {
-      value = local.tre_core_tags
+    "storage_account_resource_id" = {
+      value = azurerm_storage_account.az_monitor.id
     }
   })
 
   depends_on = [
-    azurerm_log_analytics_workspace.core,
-    azurerm_resource_group_template_deployment.app_insights_core
+    azurerm_application_insights.core
   ]
 }
 
+# Per https://docs.microsoft.com/en-us/azure/azure-monitor/profiler/profiler-bring-your-own-storage#grant-access-to-diagnostic-services-to-your-storage-account
+resource "azurerm_role_assignment" "appinsights_storage_permission" {
+  scope                = azurerm_storage_account.az_monitor.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = "6243488d-10d8-4ea0-884e-c2d5d1b7462d" # id of: Diagnostic Services Trusted Storage Access
+}
+
 resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" {
   name                = "pe-ampls-${var.tre_id}"
   resource_group_name = var.resource_group_name
   location            = var.location
   subnet_id           = var.shared_subnet_id
-  tags                = local.tre_core_tags
+  tags                = var.tre_core_tags
 
   lifecycle { ignore_changes = [tags] }
+  depends_on = [
+    azurerm_monitor_private_link_scoped_service.ampls_app_insights,
+  ]
 
   private_service_connection {
-    private_connection_resource_id = jsondecode(azurerm_resource_group_template_deployment.ampls_core.output_content).resourceId.value
+    private_connection_resource_id = azurerm_monitor_private_link_scope.ampls_core.id
     name                           = "psc-ampls-${var.tre_id}"
     subresource_names              = ["azuremonitor"]
     is_manual_connection           = false

templates/core/terraform/azure-monitor/locals.tf

@@ -1,7 +0,0 @@
-locals {
-  app_insights_name = "appi-${var.tre_id}"
-  tre_core_tags = {
-    tre_id              = var.tre_id
-    tre_core_service_id = var.tre_id
-  }
-}

templates/core/terraform/azure-monitor/outputs.tf

@@ -1,5 +1,5 @@
 output "app_insights_connection_string" {
-  value = jsondecode(azurerm_resource_group_template_deployment.app_insights_core.output_content).connectionString.value
+  value = azurerm_application_insights.core.connection_string
 }
 
 output "log_analytics_workspace_id" {