From efb9414e047bb680da232034879b902a2294dda2 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Thu, 4 Aug 2022 18:43:59 +0300 Subject: [PATCH] Refactor Azure Monitor in core (#2375) * destroy stale cicd environments * fix * fix * setup azmonitor & appinsights natively as possible * update version * updates * version change --- CHANGELOG.md | 4 +- devops/scripts/destroy_env_no_terraform.sh | 23 ++- .../core/terraform/azure-monitor/ampls.json | 64 -------- .../terraform/azure-monitor/app_insights.json | 63 -------- .../app_insights_byo_storage.json | 23 +++ .../terraform/azure-monitor/azure-monitor.tf | 141 ++++++++++-------- .../core/terraform/azure-monitor/locals.tf | 7 - .../core/terraform/azure-monitor/outputs.tf | 2 +- .../core/terraform/azure-monitor/variables.tf | 1 + templates/core/terraform/main.tf | 1 + templates/core/terraform/migrate.sh | 24 +++ templates/core/version.txt | 2 +- 12 files changed, 144 insertions(+), 211 deletions(-) delete mode 100644 templates/core/terraform/azure-monitor/ampls.json delete mode 100644 templates/core/terraform/azure-monitor/app_insights.json create mode 100644 templates/core/terraform/azure-monitor/app_insights_byo_storage.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bd6c41f8b..391da8d587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,14 +16,14 @@ ENHANCEMENTS: BUG FIXES: -* +* Azure monitor resourced provided by Terraform and don't allow ingestion over internet ([#2375](https://github.com/microsoft/AzureTRE/pull/2375)). ## 0.4.1 (August 03, 2022) **BREAKING CHANGES & MIGRATIONS**: * Guacamole workspace service configures firewall requirements with deployment pipeline ([#2371](https://github.com/microsoft/AzureTRE/pull/2371)). **Migration** is manual - update the templateVersion of `tre-shared-service-firewall` in Cosmos to `0.4.0` in order to use this capability. -* Workspace now has an AirlockManager role that has the permissions to review airlock requests ([#2349](https://github.com/microsoft/AzureTRE/pull/2349)). +* Workspace now has an AirlockManager role that has the permissions to review airlock requests ([#2349](https://github.com/microsoft/AzureTRE/pull/2349)). FEATURES: diff --git a/devops/scripts/destroy_env_no_terraform.sh b/devops/scripts/destroy_env_no_terraform.sh index f2c4fc7440..3f65bf5877 100755 --- a/devops/scripts/destroy_env_no_terraform.sh +++ b/devops/scripts/destroy_env_no_terraform.sh @@ -95,23 +95,10 @@ echo "Looking for diagnostic settings..." # using xargs to run in parallel. az resource list --resource-group "${core_tre_rg}" --query '[].[id]' -o tsv | xargs -P 10 -I {} bash -c 'delete_resource_diagnostic "{}"' +tre_id=${core_tre_rg#"rg-"} # purge keyvault if possible (makes it possible to reuse the same tre_id later) # this has to be done before we delete the resource group since we might not wait for it to complete - -# DEBUG START -# This section is to aid debugging an issue where keyvaults aren't being deleted and purged -echo "keyvault properties:" -az keyvault list --resource-group "${core_tre_rg}" --query "[].properties" -echo "keyvault purge protection evaluation result:" -az keyvault list --resource-group "${core_tre_rg}" --query "[?properties.enablePurgeProtection==``null``] | length (@)" - -if [[ -n ${SHOW_KEYVAULT_DEBUG_ON_DESTROY:-} ]]; then - az keyvault list --resource-group "${core_tre_rg}" --query "[].properties" --debug -fi -# DEBUG END - -tre_id=${core_tre_rg#"rg-"} keyvault_name="kv-${tre_id}" keyvault=$(az keyvault show --name "${keyvault_name}" --resource-group "${core_tre_rg}" || echo 0) if [ "${keyvault}" != "0" ]; then @@ -150,6 +137,14 @@ else echo "Resource group ${core_tre_rg} doesn't have a keyvault without purge protection." fi +# linked storage accounts don't get deleted with the workspace +workspace_name="log-${tre_id}" +workspace=$(az monitor log-analytics workspace show --workspace-name "${workspace_name}" --resource-group "${core_tre_rg}" || echo 0) +if [ "${workspace}" != "0" ]; then + az monitor log-analytics workspace linked-storage list -g "${core_tre_rg}" --workspace-name "${workspace_name}" -o tsv --query '[].id' \ + | xargs -P 10 -I {} az rest --method delete --uri "{}?api-version=2020-08-01" +fi + # this will find the mgmt, core resource groups as well as any workspace ones # we are reverse-sorting to first delete the workspace groups (might not be # good enough because we use no-wait sometimes) diff --git a/templates/core/terraform/azure-monitor/ampls.json b/templates/core/terraform/azure-monitor/ampls.json deleted file mode 100644 index 9092192f45..0000000000 --- a/templates/core/terraform/azure-monitor/ampls.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "private_link_scope_name": { - "type": "String" - }, - "workspace_name": { - "type": "String" - }, - "app_insights_name": { - "type": "String" - }, - "tre_core_tags": { - "type": "Object" - } - }, - "variables": {}, - "resources": [ - { - "type": "microsoft.insights/privatelinkscopes", - "apiVersion": "2021-07-01-preview", - "name": "[parameters('private_link_scope_name')]", - "location": "global", - "tags": "[parameters('tre_core_tags')]", - "properties": { - "accessModeSettings": { - "queryAccessMode":"Open", - "ingestionAccessMode":"Open" - } - } - }, - { - "type": "microsoft.insights/privatelinkscopes/scopedresources", - "apiVersion": "2019-10-17-preview", - "name": "[concat(parameters('private_link_scope_name'), '/', concat(parameters('workspace_name'), '-connection'))]", - "tags": "[parameters('tre_core_tags')]", - "dependsOn": [ - "[resourceId('microsoft.insights/privatelinkscopes', parameters('private_link_scope_name'))]" - ], - "properties": { - "linkedResourceId": "[resourceId('microsoft.operationalinsights/workspaces', parameters('workspace_name'))]" - } - }, - { - "type": "microsoft.insights/privatelinkscopes/scopedresources", - "apiVersion": "2019-10-17-preview", - "name": "[concat(parameters('private_link_scope_name'), '/', concat(parameters('app_insights_name'), '-connection'))]", - "tags": "[parameters('tre_core_tags')]", - "dependsOn": [ - "[resourceId('microsoft.insights/privatelinkscopes', parameters('private_link_scope_name'))]" - ], - "properties": { - "linkedResourceId": "[resourceId('microsoft.insights/components', parameters('app_insights_name'))]" - } - } - ], - "outputs": { - "resourceId": { - "type": "String", - "value": "[resourceId('microsoft.insights/privatelinkscopes', parameters('private_link_scope_name'))]" - } - } -} diff --git a/templates/core/terraform/azure-monitor/app_insights.json b/templates/core/terraform/azure-monitor/app_insights.json deleted file mode 100644 index d2543dea49..0000000000 --- a/templates/core/terraform/azure-monitor/app_insights.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "app_insights_name": { - "type": "String" - }, - "location": { - "type": "String" - }, - "log_analytics_workspace_id": { - "type": "String" - }, - "application_type": { - "type": "String" - }, - "storage_account_name": { - "type": "String" - }, - "tre_core_tags": { - "type": "Object" - } - }, - "variables": {}, - "resources": [ - { - "type": "microsoft.insights/components", - "apiVersion": "2020-02-02", - "name": "[parameters('app_insights_name')]", - "location": "[parameters('location')]", - "kind": "web", - "tags": "[parameters('tre_core_tags')]", - "properties": { - "Application_Type": "[parameters('application_type')]", - "SamplingPercentage": 100, - "RetentionInDays": 90, - "DisableIpMasking": false, - "WorkspaceResourceId": "[parameters('log_analytics_workspace_id')]", - "IngestionMode": "LogAnalytics", - "publicNetworkAccessForIngestion": "Enabled", - "publicNetworkAccessForQuery": "Enabled", - "DisableLocalAuth": false - } - }, - { - "name": "[concat(parameters('app_insights_name'), '/serviceprofiler')]", - "type": "microsoft.insights/components/linkedStorageAccounts", - "apiVersion": "2020-03-01-preview", - "properties": { - "linkedStorageAccount": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storage_account_name'))]" - }, - "dependsOn": [ - "[resourceId('microsoft.insights/components', parameters('app_insights_name'))]" - ] - } - ], - "outputs": { - "connectionString": { - "value": "[reference(resourceId('microsoft.insights/components', parameters('app_insights_name')), '2020-02-02').ConnectionString]", - "type": "String" - } - } -} diff --git a/templates/core/terraform/azure-monitor/app_insights_byo_storage.json b/templates/core/terraform/azure-monitor/app_insights_byo_storage.json new file mode 100644 index 0000000000..94cb2b57d5 --- /dev/null +++ b/templates/core/terraform/azure-monitor/app_insights_byo_storage.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "app_insights_name": { + "type": "String" + }, + "storage_account_resource_id": { + "type": "String" + } + }, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('app_insights_name'), '/serviceprofiler')]", + "type": "microsoft.insights/components/linkedStorageAccounts", + "apiVersion": "2020-03-01-preview", + "properties": { + "linkedStorageAccount": "[parameters('storage_account_resource_id')]" + } + } + ] +} diff --git a/templates/core/terraform/azure-monitor/azure-monitor.tf b/templates/core/terraform/azure-monitor/azure-monitor.tf index 8e1962744b..3df1920a9b 100644 --- a/templates/core/terraform/azure-monitor/azure-monitor.tf +++ b/templates/core/terraform/azure-monitor/azure-monitor.tf @@ -1,108 +1,131 @@ +# Log Analytics resource "azurerm_log_analytics_workspace" "core" { - name = "log-${var.tre_id}" - resource_group_name = var.resource_group_name - location = var.location - retention_in_days = 30 - sku = "PerGB2018" - tags = local.tre_core_tags + name = "log-${var.tre_id}" + resource_group_name = var.resource_group_name + location = var.location + retention_in_days = 30 + sku = "PerGB2018" + tags = var.tre_core_tags + internet_ingestion_enabled = false lifecycle { ignore_changes = [tags] } } -# Storage account for Application Insights +# Storage account for Azure Monitor ingestion # Because Private Link is enabled on Application Performance Management (APM), Bring Your Own Storage (BYOS) approach is required -resource "azurerm_storage_account" "app_insights" { - name = lower(replace("stappinsights${var.tre_id}", "-", "")) +resource "azurerm_storage_account" "az_monitor" { + name = lower(replace("stazmonitor${var.tre_id}", "-", "")) resource_group_name = var.resource_group_name location = var.location account_kind = "StorageV2" account_tier = "Standard" account_replication_type = "LRS" allow_nested_items_to_be_public = false - tags = local.tre_core_tags + tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } } -data "local_file" "app_insights_arm_template" { - filename = "${path.module}/app_insights.json" +resource "azurerm_log_analytics_linked_storage_account" "workspace_storage_ingestion" { + data_source_type = "ingestion" + resource_group_name = var.resource_group_name + workspace_resource_id = azurerm_log_analytics_workspace.core.id + storage_account_ids = [azurerm_storage_account.az_monitor.id] } -# Application Insights -# Deployed using ARM template, because Terraform's azurerm_application_insights does not support linked storage account -resource "azurerm_resource_group_template_deployment" "app_insights_core" { - name = local.app_insights_name +resource "azurerm_log_analytics_linked_storage_account" "workspace_storage_customlogs" { + data_source_type = "customlogs" + resource_group_name = var.resource_group_name + workspace_resource_id = azurerm_log_analytics_workspace.core.id + storage_account_ids = [azurerm_storage_account.az_monitor.id] +} + +resource "azurerm_monitor_private_link_scope" "ampls_core" { + name = "ampls-${var.tre_id}" resource_group_name = var.resource_group_name - deployment_mode = "Incremental" - template_content = data.local_file.app_insights_arm_template.content + tags = var.tre_core_tags - parameters_content = jsonencode({ - "app_insights_name" = { - value = local.app_insights_name - } - "location" = { - value = var.location - } - "log_analytics_workspace_id" = { - value = azurerm_log_analytics_workspace.core.id - } - "application_type" = { - value = "web" - } - "storage_account_name" = { - value = azurerm_storage_account.app_insights.name - } - "tre_core_tags" = { - value = local.tre_core_tags - } - }) + lifecycle { ignore_changes = [tags] } } -data "local_file" "ampls_arm_template" { - filename = "${path.module}/ampls.json" +resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" { + name = "ampls-log-anaytics-service" + resource_group_name = var.resource_group_name + scope_name = azurerm_monitor_private_link_scope.ampls_core.name + linked_resource_id = azurerm_log_analytics_workspace.core.id } -# Azure Monitor Private Link Scope -# See https://docs.microsoft.com/azure/azure-monitor/logs/private-link-security -resource "azurerm_resource_group_template_deployment" "ampls_core" { - name = "ampls-${var.tre_id}" + + +# Application Insights + +resource "azurerm_application_insights" "core" { + name = "appi-${var.tre_id}" + location = var.location + resource_group_name = var.resource_group_name + workspace_id = azurerm_log_analytics_workspace.core.id + application_type = "web" + internet_ingestion_enabled = false + force_customer_storage_for_profiler = true + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { + name = "ampls-app-insights-service" resource_group_name = var.resource_group_name - deployment_mode = "Incremental" - template_content = data.local_file.ampls_arm_template.content + scope_name = azurerm_monitor_private_link_scope.ampls_core.name + linked_resource_id = azurerm_application_insights.core.id +} + +data "local_file" "app_insights_byo_storage_arm_template" { + filename = "${path.module}/app_insights_byo_storage.json" +} +# Deployed using ARM template, because Terraform's azurerm_application_insights does not support linked storage account +# https://docs.microsoft.com/en-us/azure/azure-monitor/app/profiler-bring-your-own-storage +resource "azurerm_resource_group_template_deployment" "app_insights_byo_storage" { + name = azurerm_application_insights.core.name + resource_group_name = var.resource_group_name + deployment_mode = "Incremental" + template_content = data.local_file.app_insights_byo_storage_arm_template.content parameters_content = jsonencode({ - "private_link_scope_name" = { - value = "ampls-${var.tre_id}" - } - "workspace_name" = { - value = azurerm_log_analytics_workspace.core.name - } "app_insights_name" = { - value = local.app_insights_name + value = azurerm_application_insights.core.name } - "tre_core_tags" = { - value = local.tre_core_tags + "storage_account_resource_id" = { + value = azurerm_storage_account.az_monitor.id } }) depends_on = [ - azurerm_log_analytics_workspace.core, - azurerm_resource_group_template_deployment.app_insights_core + azurerm_application_insights.core ] } +# Per https://docs.microsoft.com/en-us/azure/azure-monitor/profiler/profiler-bring-your-own-storage#grant-access-to-diagnostic-services-to-your-storage-account +resource "azurerm_role_assignment" "appinsights_storage_permission" { + scope = azurerm_storage_account.az_monitor.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = "6243488d-10d8-4ea0-884e-c2d5d1b7462d" # id of: Diagnostic Services Trusted Storage Access +} + resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { name = "pe-ampls-${var.tre_id}" resource_group_name = var.resource_group_name location = var.location subnet_id = var.shared_subnet_id - tags = local.tre_core_tags + tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } + depends_on = [ + azurerm_monitor_private_link_scoped_service.ampls_app_insights, + ] private_service_connection { - private_connection_resource_id = jsondecode(azurerm_resource_group_template_deployment.ampls_core.output_content).resourceId.value + private_connection_resource_id = azurerm_monitor_private_link_scope.ampls_core.id name = "psc-ampls-${var.tre_id}" subresource_names = ["azuremonitor"] is_manual_connection = false diff --git a/templates/core/terraform/azure-monitor/locals.tf b/templates/core/terraform/azure-monitor/locals.tf index 11464c94d3..e69de29bb2 100644 --- a/templates/core/terraform/azure-monitor/locals.tf +++ b/templates/core/terraform/azure-monitor/locals.tf @@ -1,7 +0,0 @@ -locals { - app_insights_name = "appi-${var.tre_id}" - tre_core_tags = { - tre_id = var.tre_id - tre_core_service_id = var.tre_id - } -} diff --git a/templates/core/terraform/azure-monitor/outputs.tf b/templates/core/terraform/azure-monitor/outputs.tf index 5b70fabf7b..a568378b7a 100644 --- a/templates/core/terraform/azure-monitor/outputs.tf +++ b/templates/core/terraform/azure-monitor/outputs.tf @@ -1,5 +1,5 @@ output "app_insights_connection_string" { - value = jsondecode(azurerm_resource_group_template_deployment.app_insights_core.output_content).connectionString.value + value = azurerm_application_insights.core.connection_string } output "log_analytics_workspace_id" { diff --git a/templates/core/terraform/azure-monitor/variables.tf b/templates/core/terraform/azure-monitor/variables.tf index df528f1760..c94c2f3bb6 100644 --- a/templates/core/terraform/azure-monitor/variables.tf +++ b/templates/core/terraform/azure-monitor/variables.tf @@ -7,3 +7,4 @@ variable "azure_monitor_oms_opinsights_dns_zone_id" {} variable "azure_monitor_ods_opinsights_dns_zone_id" {} variable "azure_monitor_agentsvc_dns_zone_id" {} variable "blob_core_dns_zone_id" {} +variable "tre_core_tags" {} diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index 5d7ca9ee63..d91c28a7db 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -67,6 +67,7 @@ module "azure_monitor" { azure_monitor_ods_opinsights_dns_zone_id = module.network.azure_monitor_ods_opinsights_dns_zone_id azure_monitor_agentsvc_dns_zone_id = module.network.azure_monitor_agentsvc_dns_zone_id blob_core_dns_zone_id = module.network.blob_core_dns_zone_id + tre_core_tags = local.tre_core_tags } module "network" { diff --git a/templates/core/terraform/migrate.sh b/templates/core/terraform/migrate.sh index 30939c5271..62ecf95940 100755 --- a/templates/core/terraform/migrate.sh +++ b/templates/core/terraform/migrate.sh @@ -48,3 +48,27 @@ if [ -n "${api_app_service_id}" ]; then terraform import azurerm_linux_web_app.api "${api_app_service_id}" fi fi + +# app insights via -> native tf resource +app_insights_via_arm=$(echo "${terraform_show_json}" \ + | jq -r 'select(.values.root_module.child_modules != null) .values.root_module.child_modules[] | select (.address=="module.azure_monitor") | .resources[] | select(.address=="module.azure_monitor.azurerm_resource_group_template_deployment.app_insights_core") | .values.id') +if [ -n "${app_insights_via_arm}" ]; then + echo "Migrating ${app_insights_via_arm}" + + PLAN_FILE="tfplan$$" + TS=$(date +"%s") + LOG_FILE="${TS}-tre-core-migrate.log" + + # This variables are loaded in for us + # shellcheck disable=SC2154 + ../../../devops/scripts/terraform_wrapper.sh \ + -g "${TF_VAR_mgmt_resource_group_name}" \ + -s "${TF_VAR_mgmt_storage_account_name}" \ + -n "${TF_VAR_terraform_state_container_name}" \ + -k "${TRE_ID}" \ + -l "${LOG_FILE}" \ + -c "terraform plan -target module.azure_monitor.azurerm_resource_group_template_deployment.app_insights_core -target module.azure_monitor.azurerm_resource_group_template_deployment.ampls_core -out ${PLAN_FILE} && \ + terraform apply -input=false -auto-approve ${PLAN_FILE}" +fi + +echo "Migration is done." diff --git a/templates/core/version.txt b/templates/core/version.txt index a34b2f6b04..a3a9bd5443 100644 --- a/templates/core/version.txt +++ b/templates/core/version.txt @@ -1 +1 @@ -__version__ = "0.4.7" +__version__ = "0.4.8"