From 1de8f60d83111389ea57926c614840afea5721ef Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Fri, 30 Sep 2022 08:43:17 +0000 Subject: [PATCH 01/16] terraform init in core/ops shouldn't use -upgrade --- devops/scripts/terraform_wrapper.sh | 2 +- devops/terraform/bootstrap.sh | 2 +- devops/terraform/deploy.sh | 2 +- templates/core/terraform/.terraform.lock.hcl | 55 ++++++++++---------- templates/core/terraform/main.tf | 4 +- templates/core/terraform/migrate.sh | 9 ++-- templates/core/terraform/outputs.sh | 2 +- templates/core/version.txt | 2 +- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/devops/scripts/terraform_wrapper.sh b/devops/scripts/terraform_wrapper.sh index c011e4e57e..efa02a8078 100755 --- a/devops/scripts/terraform_wrapper.sh +++ b/devops/scripts/terraform_wrapper.sh @@ -91,7 +91,7 @@ if [[ -z ${tf_logfile+x} ]]; then echo -e "No logfile provided, using ${tf_logfile}\n" fi -terraform init -input=false -backend=true -reconfigure -upgrade \ +terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=${mgmt_resource_group_name}" \ -backend-config="storage_account_name=${mgmt_storage_account_name}" \ -backend-config="container_name=${container_name}" \ diff --git a/devops/terraform/bootstrap.sh b/devops/terraform/bootstrap.sh index dd80873322..f860b7341f 100755 --- a/devops/terraform/bootstrap.sh +++ b/devops/terraform/bootstrap.sh @@ -33,7 +33,7 @@ BOOTSTRAP_BACKEND # Set up Terraform echo -e "\n\e[34m»»» ✨ \e[96mTerraform init\e[0m..." -terraform init -input=false -backend=true -reconfigure -upgrade +terraform init -input=false -backend=true -reconfigure # Import the storage account & res group into state echo -e "\n\e[34m»»» 📤 \e[96mImporting resources to state\e[0m..." diff --git a/devops/terraform/deploy.sh b/devops/terraform/deploy.sh index f9cabcb90d..bd8c59657d 100755 --- a/devops/terraform/deploy.sh +++ b/devops/terraform/deploy.sh @@ -7,7 +7,7 @@ set -o nounset PLAN_FILE="devops.tfplan" -terraform init -input=false -backend=true -reconfigure -upgrade +terraform init -input=false -backend=true -reconfigure terraform plan -out ${PLAN_FILE} terraform apply -auto-approve ${PLAN_FILE} diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 6bfb40419c..0244cb279e 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -22,21 +22,22 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/http" { - version = "3.0.1" + version = "3.1.0" + constraints = "~> 3.1.0" hashes = [ - "h1:4N7YctkZrU+K2AvUF57c1qUvoD92bBJj6vXwf/FKMhM=", - "zh:3b161998147d8cc3986a1580ddb065009ab628747424934cbcb9d221783541f8", - "zh:62c78b565cde08d8e3b98e8138cd8e46b50fdc2ddc560ac1f62b5646ce8e9b1f", - "zh:69ba560cd6360a285e83e1c220ab140d3119371850756ff2ed0abe39d362ea49", + "h1:0QHdTeDcRFKD4YybtVl1F95/qo8n4DY5fANQVYBvt10=", + "zh:04160b9c74dfe105f64678c0521279cda6516a3b8cdb6748078318af64563faf", + "zh:2d9b4df29aab50496b6371d925d6d6b3c45788850599fd7ba553411abc9c8326", + "zh:3d36344fae7cfafabfb7fd1108916d7251dcfd550d13b129c25437b43bc2e461", + "zh:58ea39aab145edb067f0fe183c2def1bfc93b57bd9ab0289074dba511bc17644", + "zh:6e2d491f02ba4e4134ca8a8cb7312b3a691bdad80a33a29f69d58a5740fade0c", + "zh:70a8d3fa67fd5a5fb5d9baba22be01986e38dd0f84f1e40f341fe55b491b0a03", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:95f38aebfa176a3424a329bc0f2e958bcf5a1f98d91dee21a436ca670fb2d570", - "zh:97eae729eb859948201d4393761f5c1a7ffe84046473527f65163f062d9af5d9", - "zh:b42de839114707e2fcfdf5ebf3a89129e5e17ebb5f84651c5775daecd776dc3b", - "zh:c47fa93605b8378504008534e0057e295d209a2128553c7b1bcc4fc7f6efafa2", - "zh:d9d4fe5143f80c1ccf22b055f069445ab7470942bb46027dadda8f3bc62d2780", - "zh:f051820764c50f4736d21e40d9b13a1ffde678748a9e6e1ef22a26adf27db9bf", - "zh:f67c9b73998fce13e94623be9b7afe89b30e3e6d34b504f765a344b11b8808b8", - "zh:f7d255dac5a73d30c7e629699fdf064decf705cd701d29e2120cef7bf0fb1d7f", + "zh:88490f4c31bebc185f4eb7b8e3a79e3b5f92b1343f6b0c14a5c5d8c5e1de9261", + "zh:8a2ba55c5621e28faed582218213812803481765f8faea681c5c3edc61646889", + "zh:8c401d8e0c99d9733287c5ad1309692d5c7e166af6711164ad41e3579f48e45f", + "zh:ce344855648da2c575ceb7b3af18e98519d46629e6eb20358f022370745a76d2", + "zh:f9f9fe99000bc7c6b778ce23e5fe16375acad644aa1b4b4894b3cb2e9a2c7903", ] } @@ -81,22 +82,22 @@ provider "registry.terraform.io/hashicorp/null" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.3.2" - constraints = ">= 3.0.0, ~> 3.3.0" + version = "3.4.3" + constraints = ">= 3.0.0, ~> 3.4.0" hashes = [ - "h1:H5V+7iXol/EHB2+BUMzGlpIiCOdV74H8YjzCxnSAWcg=", - "zh:038293aebfede983e45ee55c328e3fde82ae2e5719c9bd233c324cfacc437f9c", - "zh:07eaeab03a723d83ac1cc218f3a59fceb7bbf301b38e89a26807d1c93c81cef8", - "zh:427611a4ce9d856b1c73bea986d841a969e4c2799c8ac7c18798d0cc42b78d32", - "zh:49718d2da653c06a70ba81fd055e2b99dfd52dcb86820a6aeea620df22cd3b30", - "zh:5574828d90b19ab762604c6306337e6cd430e65868e13ef6ddb4e25ddb9ad4c0", - "zh:7222e16f7833199dabf1bc5401c56d708ec052b2a5870988bc89ff85b68a5388", + "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:b1b2d7d934784d2aee98b0f8f07a8ccfc0410de63493ae2bf2222c165becf938", - "zh:b8f85b6a20bd264fcd0814866f415f0a368d1123cd7879c8ebbf905d370babc8", - "zh:c3813133acc02bbebddf046d9942e8ba5c35fc99191e3eb057957dafc2929912", - "zh:e7a41dbc919d1de800689a81c240c27eec6b9395564630764ebb323ea82ac8a9", - "zh:ee6d23208449a8eaa6c4f203e33f5176fa795b4b9ecf32903dffe6e2574732c2", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", ] } diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index c16d659d83..f26e8fd24f 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -7,7 +7,7 @@ terraform { } random = { source = "hashicorp/random" - version = "~> 3.3.0" + version = "~> 3.4.0" } template = { source = "hashicorp/template" @@ -19,7 +19,7 @@ terraform { } http = { source = "hashicorp/http" - version = "~> 3.0.0" + version = "~> 3.1.0" } } diff --git a/templates/core/terraform/migrate.sh b/templates/core/terraform/migrate.sh index 551fa7838e..55234b7bb5 100755 --- a/templates/core/terraform/migrate.sh +++ b/templates/core/terraform/migrate.sh @@ -7,13 +7,16 @@ set -o nounset # This variables are loaded in for us # shellcheck disable=SC2154 -terraform init -input=false -backend=true -reconfigure -upgrade \ +terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ -backend-config="key=${TRE_ID}" -echo "*** Migrating TF Resources ***" +echo "*** Migrating TF Resources... ***" +# terraform show might fail if provider schema has changed. Since we don't call apply at this stage a refresh is needed +terraform refresh -target=module.resource_processor_vmss_porter[0].random_password.password # When moving to 3.4.* + # 1. Check we have a root_module in state # 2. Grab the Resource ID # 3. Delete the old resource from state @@ -153,4 +156,4 @@ if [ -n "${api_vnet_integration}" ]; then terraform apply -input=false -auto-approve ${PLAN_FILE}" fi -echo "Migration is done." +echo "*** Migration is done. ***" diff --git a/templates/core/terraform/outputs.sh b/templates/core/terraform/outputs.sh index ee03c5f849..c6b7d08af4 100755 --- a/templates/core/terraform/outputs.sh +++ b/templates/core/terraform/outputs.sh @@ -5,7 +5,7 @@ if [ ! -f ../tre_output.json ]; then # Connect to the remote backend of Terraform export TF_LOG="" # shellcheck disable=SC2154 - terraform init -input=false -backend=true -reconfigure -upgrade \ + terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ diff --git a/templates/core/version.txt b/templates/core/version.txt index 4fef01e0d4..7fe0489074 100644 --- a/templates/core/version.txt +++ b/templates/core/version.txt @@ -1 +1 @@ -__version__ = "0.4.28" +__version__ = "0.4.29" From 649186c918fd2c23fea273db561a71b09920f92f Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Fri, 30 Sep 2022 08:44:21 +0000 Subject: [PATCH 02/16] update terraform in devcontainer --- .devcontainer/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7cb862150b..c861b39b4e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -24,7 +24,8 @@ ARG NODE_VERSION="lts/*" RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1" # Install terraform -ARG TERRAFORM_VERSION="1.2.7" +# version 1.3.0/1 has issues with keep recreating certificate +ARG TERRAFORM_VERSION="1.2.9" COPY .devcontainer/scripts/terraform.sh /tmp/ RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin @@ -40,15 +41,13 @@ RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release && apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io --no-install-recommends \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# Install nekos act - run GitHub workflows locally https://github.com/nektos/act -RUN if [ "${INTERACTIVE}" = "true" ]; then curl https://raw.githubusercontent.com/nektos/act/master/install.sh | bash; fi - # Install Certbot -RUN apt-get update && apt-get install -y python3 python3-venv libaugeas0 --no-install-recommends \ +RUN if [ "${INTERACTIVE}" = "true" ]; then \ + apt-get update && apt-get install -y python3 python3-venv libaugeas0 --no-install-recommends \ && python3 -m venv /opt/certbot/ \ && /opt/certbot/bin/pip install --no-cache-dir --upgrade pip \ && /opt/certbot/bin/pip install --no-cache-dir certbot \ - && apt-get clean -y && rm -rf /var/lib/apt/lists/* + && apt-get clean -y && rm -rf /var/lib/apt/lists/* ; fi # Install Porter # Not using the script from https://cdn.porter.sh/latest/install-linux.sh From 6a062a18f95ddc7adde08b8ebd6ec5290fc446ce Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:07:51 +0000 Subject: [PATCH 03/16] always refresh --- templates/core/terraform/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/migrate.sh b/templates/core/terraform/migrate.sh index 55234b7bb5..ab677e32cf 100755 --- a/templates/core/terraform/migrate.sh +++ b/templates/core/terraform/migrate.sh @@ -15,7 +15,7 @@ terraform init -input=false -backend=true -reconfigure \ echo "*** Migrating TF Resources... ***" # terraform show might fail if provider schema has changed. Since we don't call apply at this stage a refresh is needed -terraform refresh -target=module.resource_processor_vmss_porter[0].random_password.password # When moving to 3.4.* +terraform refresh # 1. Check we have a root_module in state # 2. Grab the Resource ID From 13f356a81d448ea27da08aac81df2ad402a2db8b Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 06:16:28 +0000 Subject: [PATCH 04/16] tflint --- templates/core/terraform/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index f26e8fd24f..b404db968b 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -9,10 +9,6 @@ terraform { source = "hashicorp/random" version = "~> 3.4.0" } - template = { - source = "hashicorp/template" - version = "~> 2.2.0" - } local = { source = "hashicorp/local" version = "~> 2.2.0" From 5484475c54ae4a561f307a1473fe088fb0ae0c02 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 06:29:19 +0000 Subject: [PATCH 05/16] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index caaf8dd129..43c2b2b259 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ BUG FIXES: * Resource processor error on deploying user-resource: TypeError: 'NoneType' object is not iterable ([#2569](https://github.com/microsoft/AzureTRE/issues/2569)) * Update Porter and Terraform mixin versions ([#2639](https://github.com/microsoft/AzureTRE/issues/2639)) * Airlock Manager should have permissions to get SAS token ([#2502](https://github.com/microsoft/AzureTRE/issues/2502)) +* Terraform unmarshal errors in `migrate.sh` ([#2673](https://github.com/microsoft/AzureTRE/issues/2673)) ## 0.4.3 (September 12, 2022) From 51870653241270d52ea5b21530bae23b999c4aab Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 06:40:51 +0000 Subject: [PATCH 06/16] Remove doker env variables --- CHANGELOG.md | 1 + templates/core/terraform/api-webapp.tf | 2 +- templates/core/terraform/deploy.sh | 7 ----- templates/core/terraform/destroy.sh | 14 +++++----- templates/core/terraform/import.sh | 26 ++++++++++--------- templates/core/terraform/locals.tf | 1 + templates/core/terraform/main.tf | 4 +-- templates/core/terraform/outputs.tf | 2 +- templates/core/terraform/variables.tf | 5 ---- templates/core/version.txt | 2 +- .../admin-vm/terraform/deploy.sh | 11 +------- .../admin-vm/terraform/destroy.sh | 10 +------ .../shared_services/gitea/terraform/deploy.sh | 13 +++++++--- .../gitea/terraform/destroy.sh | 13 +++++++--- .../gitea/terraform/deploy.sh | 16 +++++------- .../gitea/terraform/destroy.sh | 15 +++++------ 16 files changed, 62 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43c2b2b259..4ebe045a1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ ENHANCEMENTS: * Cancelling an Airlock request triggers deletion of the request container and files ([#2584](https://github.com/microsoft/AzureTRE/pull/2584)) * Move admin-vm from core to a shared service ([#2624](https://github.com/microsoft/AzureTRE/pull/2624)) +* Remove obsolete docker environment variables ([#TBD](https://github.com/microsoft/AzureTRE/pull/TBD)) BUG FIXES: diff --git a/templates/core/terraform/api-webapp.tf b/templates/core/terraform/api-webapp.tf index 7ecf84cd6e..32b4a4b17c 100644 --- a/templates/core/terraform/api-webapp.tf +++ b/templates/core/terraform/api-webapp.tf @@ -73,7 +73,7 @@ resource "azurerm_linux_web_app" "api" { ftps_state = "Disabled" application_stack { - docker_image = "${var.docker_registry_server}/${var.api_image_repository}" + docker_image = "${local.docker_registry_server}/${var.api_image_repository}" docker_image_tag = local.version } diff --git a/templates/core/terraform/deploy.sh b/templates/core/terraform/deploy.sh index f9d15df754..84f240cd5f 100755 --- a/templates/core/terraform/deploy.sh +++ b/templates/core/terraform/deploy.sh @@ -5,13 +5,6 @@ set -o pipefail set -o nounset # set -o xtrace -# This variables are loaded in for us -# shellcheck disable=SC2154 -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username="${TF_VAR_acr_name}" -TF_VAR_docker_registry_password=$(az acr credential show --name "${TF_VAR_acr_name}" --query passwords[0].value -o tsv | sed 's/"//g') -export TF_VAR_docker_registry_password - # This is where we can migrate any Terraform before we plan and apply # For instance deprecated Terraform resources ./migrate.sh diff --git a/templates/core/terraform/destroy.sh b/templates/core/terraform/destroy.sh index a264098e70..6f6892e589 100755 --- a/templates/core/terraform/destroy.sh +++ b/templates/core/terraform/destroy.sh @@ -5,11 +5,9 @@ set -o pipefail set -o nounset # set -o xtrace -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -export TF_VAR_docker_registry_password=$(az acr credential show --name ${TF_VAR_acr_name} --query passwords[0].value -o tsv | sed 's/"//g') - -../../../devops/scripts/terraform_wrapper.sh -g $TF_VAR_mgmt_resource_group_name \ - -s $TF_VAR_mgmt_storage_account_name \ - -n $TF_VAR_terraform_state_container_name \ - -k $TRE_ID -c "terraform destroy -auto-approve" +# This variables are loaded in for us +# shellcheck disable=SC2154 +../../../devops/scripts/terraform_wrapper.sh -g "${TF_VAR_mgmt_resource_group_name}" \ + -s "${TF_VAR_mgmt_storage_account_name}" \ + -n "${TF_VAR_terraform_state_container_name}" \ + -k "${TRE_ID}" -c "terraform destroy -auto-approve" diff --git a/templates/core/terraform/import.sh b/templates/core/terraform/import.sh index 44f4352937..7b3801c1e7 100755 --- a/templates/core/terraform/import.sh +++ b/templates/core/terraform/import.sh @@ -1,15 +1,17 @@ -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -export TF_VAR_docker_registry_password=$(az acr credential show --name ${TF_VAR_acr_name} --query passwords[0].value -o tsv | sed 's/"//g') +#!/bin/bash -export TF_LOG="" - -cd ./templates/core/terraform/ +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace -terraform init -input=false -backend=true -reconfigure -upgrade \ - -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ - -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ - -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=${TRE_ID}" +export TF_LOG="" -terraform import ... +# This variables are loaded in for us +# shellcheck disable=SC2154 +../../../devops/scripts/terraform_wrapper.sh \ + -g "${TF_VAR_mgmt_resource_group_name}" \ + -s "${TF_VAR_mgmt_storage_account_name}" \ + -n "${TF_VAR_terraform_state_container_name}" \ + -k "${TRE_ID}" \ + -c "terraform import ..." diff --git a/templates/core/terraform/locals.tf b/templates/core/terraform/locals.tf index a3cf7b88e1..118442d0c9 100644 --- a/templates/core/terraform/locals.tf +++ b/templates/core/terraform/locals.tf @@ -8,4 +8,5 @@ locals { "AppServiceHTTPLogs", "AppServiceConsoleLogs", "AppServiceAppLogs", "AppServiceFileAuditLogs", "AppServiceAuditLogs", "AppServiceIPSecAuditLogs", "AppServicePlatformLogs", "AppServiceAntivirusScanAuditLogs" ] + docker_registry_server = "${var.acr_name}.azurecr.io" } diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index b404db968b..91f82b938b 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -105,7 +105,7 @@ module "airlock_resources" { resource_group_name = azurerm_resource_group.core.name airlock_storage_subnet_id = module.network.airlock_storage_subnet_id airlock_events_subnet_id = module.network.airlock_events_subnet_id - docker_registry_server = var.docker_registry_server + docker_registry_server = local.docker_registry_server mgmt_resource_group_name = var.mgmt_resource_group_name mgmt_acr_name = var.acr_name api_principal_id = azurerm_user_assigned_identity.id.principal_id @@ -135,7 +135,7 @@ module "resource_processor_vmss_porter" { acr_id = data.azurerm_container_registry.mgmt_acr.id app_insights_connection_string = module.azure_monitor.app_insights_connection_string resource_processor_subnet_id = module.network.resource_processor_subnet_id - docker_registry_server = var.docker_registry_server + docker_registry_server = local.docker_registry_server resource_processor_vmss_porter_image_repository = var.resource_processor_vmss_porter_image_repository service_bus_namespace_id = azurerm_servicebus_namespace.sb.id service_bus_resource_request_queue = azurerm_servicebus_queue.workspacequeue.name diff --git a/templates/core/terraform/outputs.tf b/templates/core/terraform/outputs.tf index 34d38c372f..d435f095ac 100644 --- a/templates/core/terraform/outputs.tf +++ b/templates/core/terraform/outputs.tf @@ -69,7 +69,7 @@ output "terraform_state_container_name" { } output "registry_server" { - value = var.docker_registry_server + value = local.docker_registry_server } output "event_grid_status_changed_topic_endpoint" { diff --git a/templates/core/terraform/variables.tf b/templates/core/terraform/variables.tf index 0dc605fd6d..cbf5106e2f 100644 --- a/templates/core/terraform/variables.tf +++ b/templates/core/terraform/variables.tf @@ -65,11 +65,6 @@ variable "resource_processor_number_processes_per_instance" { description = "The number of CPU processes to run the RP on per VM instance" } -variable "docker_registry_server" { - type = string - description = "Docker registry server" -} - variable "swagger_ui_client_id" { type = string description = "The client id (app id) of the registration in Azure AD for the Swagger UI" diff --git a/templates/core/version.txt b/templates/core/version.txt index 7fe0489074..e2b01a98c0 100644 --- a/templates/core/version.txt +++ b/templates/core/version.txt @@ -1 +1 @@ -__version__ = "0.4.29" +__version__ = "0.4.31" diff --git a/templates/shared_services/admin-vm/terraform/deploy.sh b/templates/shared_services/admin-vm/terraform/deploy.sh index f15ba8a684..08a52b7119 100755 --- a/templates/shared_services/admin-vm/terraform/deploy.sh +++ b/templates/shared_services/admin-vm/terraform/deploy.sh @@ -6,15 +6,6 @@ set -o nounset # Uncomment this line to see each command for debugging (careful: this will show secrets!) # set -o xtrace - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username="${TF_VAR_acr_name}" -TF_VAR_docker_registry_password=$(az acr credential show --name "${TF_VAR_acr_name}" --query passwords[0].value -o tsv | sed 's/"//g') -export TF_VAR_docker_registry_password - export TF_LOG="" # This script assumes you have created an .env from the sample and the variables @@ -24,6 +15,6 @@ terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=tre-workspace-service-gitea-$TF_VAR_id" + -backend-config="key=${TRE_ID}-adminvm" terraform plan terraform apply -auto-approve diff --git a/templates/shared_services/admin-vm/terraform/destroy.sh b/templates/shared_services/admin-vm/terraform/destroy.sh index 8ae3e7a479..b74684a968 100755 --- a/templates/shared_services/admin-vm/terraform/destroy.sh +++ b/templates/shared_services/admin-vm/terraform/destroy.sh @@ -6,14 +6,6 @@ set -o nounset # Uncomment this line to see each command for debugging (careful: this will show secrets!) # set -o xtrace -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -TF_VAR_docker_registry_password=$(az acr credential show --name "${TF_VAR_acr_name}" --query passwords[0].value -o tsv | sed 's/"//g') -export TF_VAR_docker_registry_password - export TF_LOG="" # This script assumes you have created an .env from the sample and the variables @@ -23,6 +15,6 @@ terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=tre-workspace-service-gitea-$TF_VAR_id" + -backend-config="key=${TRE_ID}-adminvm" terraform destroy -auto-approve diff --git a/templates/shared_services/gitea/terraform/deploy.sh b/templates/shared_services/gitea/terraform/deploy.sh index fb70897d72..3559959fca 100755 --- a/templates/shared_services/gitea/terraform/deploy.sh +++ b/templates/shared_services/gitea/terraform/deploy.sh @@ -1,9 +1,16 @@ -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -export TF_VAR_docker_registry_password=$(az acr credential show --name ${TF_VAR_acr_name} --query passwords[0].value -o tsv | sed 's/"//g') +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset +# Uncomment this line to see each command for debugging (careful: this will show secrets!) +# set -o xtrace export TF_LOG="" +# This script assumes you have created an .env from the sample and the variables +# will come from there. +# shellcheck disable=SC2154 terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ diff --git a/templates/shared_services/gitea/terraform/destroy.sh b/templates/shared_services/gitea/terraform/destroy.sh index 1c22951ecd..58d56580b8 100755 --- a/templates/shared_services/gitea/terraform/destroy.sh +++ b/templates/shared_services/gitea/terraform/destroy.sh @@ -1,9 +1,16 @@ -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -export TF_VAR_docker_registry_password=$(az acr credential show --name ${TF_VAR_acr_name} --query passwords[0].value -o tsv | sed 's/"//g') +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset +# Uncomment this line to see each command for debugging (careful: this will show secrets!) +# set -o xtrace export TF_LOG="" +# This script assumes you have created an .env from the sample and the variables +# will come from there. +# shellcheck disable=SC2154 terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ diff --git a/templates/workspace_services/gitea/terraform/deploy.sh b/templates/workspace_services/gitea/terraform/deploy.sh index 0d43b987d8..0d9d3cc915 100755 --- a/templates/workspace_services/gitea/terraform/deploy.sh +++ b/templates/workspace_services/gitea/terraform/deploy.sh @@ -1,13 +1,10 @@ #!/bin/bash -set -e -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -TF_VAR_docker_registry_password=$(az acr credential show --name "${TF_VAR_acr_name}" --query passwords[0].value -o tsv | sed 's/"//g') -export TF_VAR_docker_registry_password +set -o errexit +set -o pipefail +set -o nounset +# Uncomment this line to see each command for debugging (careful: this will show secrets!) +# set -o xtrace export TF_LOG="" @@ -18,6 +15,7 @@ terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=tre-workspace-service-gitea-$TF_VAR_id" + -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}" + terraform plan terraform apply -auto-approve diff --git a/templates/workspace_services/gitea/terraform/destroy.sh b/templates/workspace_services/gitea/terraform/destroy.sh index 664f1efac6..bdf0202e1c 100755 --- a/templates/workspace_services/gitea/terraform/destroy.sh +++ b/templates/workspace_services/gitea/terraform/destroy.sh @@ -1,13 +1,10 @@ #!/bin/bash -set -e -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -export TF_VAR_docker_registry_server="$TF_VAR_acr_name.azurecr.io" -export TF_VAR_docker_registry_username=$TF_VAR_acr_name -TF_VAR_docker_registry_password=$(az acr credential show --name "${TF_VAR_acr_name}" --query passwords[0].value -o tsv | sed 's/"//g') -export TF_VAR_docker_registry_password +set -o errexit +set -o pipefail +set -o nounset +# Uncomment this line to see each command for debugging (careful: this will show secrets!) +# set -o xtrace export TF_LOG="" @@ -18,6 +15,6 @@ terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=tre-workspace-service-gitea-$TF_VAR_id" + -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}" terraform destroy -auto-approve From 59f5e66ae036c0753ae8ab3b3c2ac57dca2fd1cd Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 06:48:00 +0000 Subject: [PATCH 07/16] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ebe045a1f..4a184c13a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ ENHANCEMENTS: * Cancelling an Airlock request triggers deletion of the request container and files ([#2584](https://github.com/microsoft/AzureTRE/pull/2584)) * Move admin-vm from core to a shared service ([#2624](https://github.com/microsoft/AzureTRE/pull/2624)) -* Remove obsolete docker environment variables ([#TBD](https://github.com/microsoft/AzureTRE/pull/TBD)) +* Remove obsolete docker environment variables ([#2675](https://github.com/microsoft/AzureTRE/pull/2675)) BUG FIXES: From 1a49b04dce66cf102bc17f23f17adadf332d5178 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 09:39:57 +0000 Subject: [PATCH 08/16] align bundles with new terraform mixin --- resource_processor/_version.py | 2 +- resource_processor/scripts/porter.sh | 2 +- resource_processor/vmss_porter/Dockerfile | 4 +++- templates/shared_services/admin-vm/Dockerfile.tmpl | 7 ------- templates/shared_services/admin-vm/porter.yaml | 2 +- .../shared_services/airlock_notifier/Dockerfile.tmpl | 7 ------- templates/shared_services/airlock_notifier/porter.yaml | 2 +- templates/shared_services/certs/Dockerfile.tmpl | 7 ------- templates/shared_services/certs/porter.yaml | 2 +- templates/shared_services/cyclecloud/Dockerfile.tmpl | 7 ------- templates/shared_services/cyclecloud/porter.yaml | 2 +- templates/shared_services/firewall/Dockerfile.tmpl | 7 ------- templates/shared_services/firewall/porter.yaml | 2 +- templates/shared_services/gitea/Dockerfile.tmpl | 7 ------- templates/shared_services/gitea/porter.yaml | 2 +- .../shared_services/sonatype-nexus-vm/Dockerfile.tmpl | 7 ------- templates/shared_services/sonatype-nexus-vm/porter.yaml | 2 +- templates/workspace_services/azureml/Dockerfile.tmpl | 7 ------- templates/workspace_services/azureml/porter.yaml | 2 +- .../azureml/user_resources/aml_compute/Dockerfile.tmpl | 6 ------ .../azureml/user_resources/aml_compute/porter.yaml | 2 +- templates/workspace_services/gitea/Dockerfile.tmpl | 7 ------- templates/workspace_services/gitea/porter.yaml | 2 +- templates/workspace_services/guacamole/Dockerfile.tmpl | 7 ------- templates/workspace_services/guacamole/porter.yaml | 2 +- .../guacamole-azure-import-reviewvm/Dockerfile.tmpl | 7 ------- .../guacamole-azure-import-reviewvm/porter.yaml | 2 +- .../guacamole-azure-linuxvm/Dockerfile.tmpl | 7 ------- .../user_resources/guacamole-azure-linuxvm/porter.yaml | 2 +- .../guacamole-azure-windowsvm/Dockerfile.tmpl | 7 ------- .../user_resources/guacamole-azure-windowsvm/porter.yaml | 2 +- templates/workspace_services/innereye/Dockerfile.tmpl | 7 ------- templates/workspace_services/innereye/porter.yaml | 2 +- templates/workspace_services/mlflow/Dockerfile.tmpl | 7 ------- templates/workspace_services/mlflow/porter.yaml | 2 +- templates/workspace_services/mysql/Dockerfile.tmpl | 7 ------- templates/workspace_services/mysql/porter.yaml | 2 +- .../workspaces/airlock-import-review/Dockerfile.tmpl | 7 ------- templates/workspaces/airlock-import-review/porter.yaml | 2 +- templates/workspaces/base/Dockerfile.tmpl | 7 ------- templates/workspaces/base/porter.yaml | 2 +- templates/workspaces/unrestricted/Dockerfile.tmpl | 8 -------- templates/workspaces/unrestricted/porter.yaml | 2 +- 43 files changed, 25 insertions(+), 163 deletions(-) diff --git a/resource_processor/_version.py b/resource_processor/_version.py index a34b2f6b04..a3a9bd5443 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.4.7" +__version__ = "0.4.8" diff --git a/resource_processor/scripts/porter.sh b/resource_processor/scripts/porter.sh index 92e76c2ddf..4fc61dab15 100755 --- a/resource_processor/scripts/porter.sh +++ b/resource_processor/scripts/porter.sh @@ -11,7 +11,7 @@ chmod +x "${PORTER_HOME}/porter" ln -s "${PORTER_HOME}/porter" "${PORTER_HOME}/runtimes/porter-runtime" "${PORTER_HOME}/porter" mixin install exec --version "${PORTER_PKG_PERMALINK}" -"${PORTER_HOME}/porter" mixin install terraform --version "${PORTER_PKG_PERMALINK}" +"${PORTER_HOME}/porter" mixin install terraform --version "${PORTER_TERRAFORM_MIXIN_PKG_PERMALINK}" "${PORTER_HOME}/porter" mixin install az --version "${PORTER_PKG_PERMALINK}" "${PORTER_HOME}/porter" plugin install azure --version "${PORTER_PKG_PERMALINK}" "${PORTER_HOME}/porter" mixin install docker --version "${PORTER_PKG_PERMALINK}" diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index eb9b29c92a..e0475ae523 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.8-slim-buster SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install Azure CLI -ARG AZURE_CLI_VERSION=2.39.0-1~buster +ARG AZURE_CLI_VERSION=2.40.0-1~buster COPY scripts/azure-cli.sh /tmp/ RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \ && /tmp/azure-cli.sh @@ -12,11 +12,13 @@ RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \ ARG PORTER_MIRROR=https://cdn.porter.sh ARG PORTER_PERMALINK=v0.38.13 ARG PORTER_PKG_PERMALINK=latest +ARG PORTER_TERRAFORM_MIXIN_PKG_PERMALINK="v1.0.0-rc.1" ARG PORTER_HOME=/root/.porter/ COPY scripts/porter.sh /tmp/ RUN export PORTER_MIRROR=${PORTER_MIRROR} \ PORTER_PERMALINK=${PORTER_PERMALINK} \ PORTER_PKG_PERMALINK=${PORTER_PKG_PERMALINK} \ + PORTER_TERRAFORM_MIXIN_PKG_PERMALINK=${PORTER_TERRAFORM_MIXIN_PKG_PERMALINK} \ PORTER_HOME=${PORTER_HOME} \ && /tmp/porter.sh diff --git a/templates/shared_services/admin-vm/Dockerfile.tmpl b/templates/shared_services/admin-vm/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/shared_services/admin-vm/Dockerfile.tmpl +++ b/templates/shared_services/admin-vm/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/shared_services/admin-vm/porter.yaml b/templates/shared_services/admin-vm/porter.yaml index 50ae58f944..e0520fbffb 100644 --- a/templates/shared_services/admin-vm/porter.yaml +++ b/templates/shared_services/admin-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-admin-vm -version: 0.1.0 +version: 0.2.0 description: "An admin vm shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/airlock_notifier/Dockerfile.tmpl b/templates/shared_services/airlock_notifier/Dockerfile.tmpl index 48fb9e7a60..25e675ade9 100644 --- a/templates/shared_services/airlock_notifier/Dockerfile.tmpl +++ b/templates/shared_services/airlock_notifier/Dockerfile.tmpl @@ -24,12 +24,5 @@ RUN apt-get update \ # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins - WORKDIR $BUNDLE_DIR/app RUN zip -r /cnab/app/LogicApp.zip . diff --git a/templates/shared_services/airlock_notifier/porter.yaml b/templates/shared_services/airlock_notifier/porter.yaml index 2af737ecc2..aa122ab8a7 100644 --- a/templates/shared_services/airlock_notifier/porter.yaml +++ b/templates/shared_services/airlock_notifier/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-airlock-notifier -version: 0.1.2 +version: 0.2.0 description: "A shared service notifying on Airlock Operations" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 9c51e4cfb0..1a6a64e024 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -38,10 +38,3 @@ RUN apt-get update \ # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 3c480a365d..be68dbf7d2 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-certs -version: 0.1.4 +version: 0.2.0 description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/shared_services/cyclecloud/Dockerfile.tmpl b/templates/shared_services/cyclecloud/Dockerfile.tmpl index 3481bc71e4..000f6a1471 100644 --- a/templates/shared_services/cyclecloud/Dockerfile.tmpl +++ b/templates/shared_services/cyclecloud/Dockerfile.tmpl @@ -24,10 +24,3 @@ RUN apt-get update \ # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/shared_services/cyclecloud/porter.yaml b/templates/shared_services/cyclecloud/porter.yaml index 3942cce12e..5a6b20ce76 100644 --- a/templates/shared_services/cyclecloud/porter.yaml +++ b/templates/shared_services/cyclecloud/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-cyclecloud -version: 0.2.7 +version: 0.3.0 description: "An Azure TRE Shared Service Template for Azure Cyclecloud" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/shared_services/firewall/Dockerfile.tmpl b/templates/shared_services/firewall/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/shared_services/firewall/Dockerfile.tmpl +++ b/templates/shared_services/firewall/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/shared_services/firewall/porter.yaml b/templates/shared_services/firewall/porter.yaml index 9ec23a52b4..6a8f0ddfbd 100644 --- a/templates/shared_services/firewall/porter.yaml +++ b/templates/shared_services/firewall/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-firewall -version: 0.5.0 +version: 0.6.0 description: "An Azure TRE Firewall shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/gitea/Dockerfile.tmpl b/templates/shared_services/gitea/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/shared_services/gitea/Dockerfile.tmpl +++ b/templates/shared_services/gitea/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/shared_services/gitea/porter.yaml b/templates/shared_services/gitea/porter.yaml index 18b5017ad0..053204e612 100644 --- a/templates/shared_services/gitea/porter.yaml +++ b/templates/shared_services/gitea/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-gitea -version: 0.3.14 +version: 0.4.0 description: "A Gitea shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/sonatype-nexus-vm/Dockerfile.tmpl b/templates/shared_services/sonatype-nexus-vm/Dockerfile.tmpl index 9de35f1044..08e4cad05a 100644 --- a/templates/shared_services/sonatype-nexus-vm/Dockerfile.tmpl +++ b/templates/shared_services/sonatype-nexus-vm/Dockerfile.tmpl @@ -19,10 +19,3 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index fff0c6378e..8695282936 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-sonatype-nexus -version: 2.1.7 +version: 2.2.0 description: "A Sonatype Nexus shared service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/azureml/Dockerfile.tmpl b/templates/workspace_services/azureml/Dockerfile.tmpl index 73ada5230e..01bb786ac0 100644 --- a/templates/workspace_services/azureml/Dockerfile.tmpl +++ b/templates/workspace_services/azureml/Dockerfile.tmpl @@ -31,10 +31,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index e56a5acf24..90cd97448f 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-azureml -version: 0.4.12 +version: 0.5.0 description: "An Azure TRE service for Azure Machine Learning" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/azureml/user_resources/aml_compute/Dockerfile.tmpl b/templates/workspace_services/azureml/user_resources/aml_compute/Dockerfile.tmpl index af0e72a2b8..86fc334c03 100644 --- a/templates/workspace_services/azureml/user_resources/aml_compute/Dockerfile.tmpl +++ b/templates/workspace_services/azureml/user_resources/aml_compute/Dockerfile.tmpl @@ -6,10 +6,4 @@ COPY . $BUNDLE_DIR # PORTER_MIXINS -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && terraform providers mirror /usr/local/share/terraform/plugins - WORKDIR $BUNDLE_DIR diff --git a/templates/workspace_services/azureml/user_resources/aml_compute/porter.yaml b/templates/workspace_services/azureml/user_resources/aml_compute/porter.yaml index e9abd6f2da..99f140df01 100644 --- a/templates/workspace_services/azureml/user_resources/aml_compute/porter.yaml +++ b/templates/workspace_services/azureml/user_resources/aml_compute/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-user-resource-aml-compute-instance -version: 0.3.2 +version: 0.4.0 description: "Azure Machine Learning Compute Instance" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/gitea/Dockerfile.tmpl b/templates/workspace_services/gitea/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/workspace_services/gitea/Dockerfile.tmpl +++ b/templates/workspace_services/gitea/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/gitea/porter.yaml b/templates/workspace_services/gitea/porter.yaml index 23b61e71c2..21ab5eb2f7 100644 --- a/templates/workspace_services/gitea/porter.yaml +++ b/templates/workspace_services/gitea/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-service-gitea -version: 0.4.0 +version: 0.5.0 description: "A Gitea workspace service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/Dockerfile.tmpl b/templates/workspace_services/guacamole/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/workspace_services/guacamole/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/guacamole/porter.yaml b/templates/workspace_services/guacamole/porter.yaml index 81da73309c..c254e29778 100644 --- a/templates/workspace_services/guacamole/porter.yaml +++ b/templates/workspace_services/guacamole/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole -version: 0.4.6 +version: 0.5.0 description: "An Azure TRE service for Guacamole" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl index 34a5df4d66..429c810ece 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl @@ -32,10 +32,3 @@ COPY . $BUNDLE_DIR # Apply patch with the difference from the base workspace RUN patch -p0 < $BUNDLE_DIR/windowsvm.diff - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml index 43b4fd482f..24c968168a 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-import-reviewvm -version: 0.0.3 +version: 0.1.0 description: "An Azure TRE User Resource Template for reviewing Airlock import requests" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 1cd6223ba3..d0ede19a89 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-linuxvm -version: 0.4.16 +version: 0.5.0 description: "An Azure TRE User Resource Template for Guacamole (Linux)" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 5d7092b635..e6a487ccf0 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-windowsvm -version: 0.4.14 +version: 0.5.0 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/innereye/Dockerfile.tmpl b/templates/workspace_services/innereye/Dockerfile.tmpl index 4c7acb6c32..7b24eb34b8 100644 --- a/templates/workspace_services/innereye/Dockerfile.tmpl +++ b/templates/workspace_services/innereye/Dockerfile.tmpl @@ -38,10 +38,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/innereye/porter.yaml b/templates/workspace_services/innereye/porter.yaml index bb1f6821b4..2c027ae625 100644 --- a/templates/workspace_services/innereye/porter.yaml +++ b/templates/workspace_services/innereye/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-innereye -version: 0.3.5 +version: 0.4.0 description: "An Azure TRE service for InnerEye Deep Learning" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/mlflow/Dockerfile.tmpl b/templates/workspace_services/mlflow/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/workspace_services/mlflow/Dockerfile.tmpl +++ b/templates/workspace_services/mlflow/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/mlflow/porter.yaml b/templates/workspace_services/mlflow/porter.yaml index 2d28f13386..450e523d85 100644 --- a/templates/workspace_services/mlflow/porter.yaml +++ b/templates/workspace_services/mlflow/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-mlflow -version: 0.3.8 +version: 0.4.0 description: "An Azure TRE service for MLflow machine learning lifecycle" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/mysql/Dockerfile.tmpl b/templates/workspace_services/mysql/Dockerfile.tmpl index 9e826e348d..8faf11cac1 100644 --- a/templates/workspace_services/mysql/Dockerfile.tmpl +++ b/templates/workspace_services/mysql/Dockerfile.tmpl @@ -17,10 +17,3 @@ ARG BUNDLE_DIR # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspace_services/mysql/porter.yaml b/templates/workspace_services/mysql/porter.yaml index 84a057fe94..510e661021 100644 --- a/templates/workspace_services/mysql/porter.yaml +++ b/templates/workspace_services/mysql/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-service-mysql -version: 0.1.2 +version: 0.2.0 description: "A MySQL workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspaces/airlock-import-review/Dockerfile.tmpl b/templates/workspaces/airlock-import-review/Dockerfile.tmpl index 435dd51a8a..dbc86d9b30 100644 --- a/templates/workspaces/airlock-import-review/Dockerfile.tmpl +++ b/templates/workspaces/airlock-import-review/Dockerfile.tmpl @@ -34,10 +34,3 @@ COPY . $BUNDLE_DIR # Apply patch with the difference from the base workspace RUN patch -p0 < $BUNDLE_DIR/workspace_base.diff - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index 228500f8f1..cab8360497 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-airlock-import-review -version: 0.3.30 +version: 0.4.0 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/Dockerfile.tmpl b/templates/workspaces/base/Dockerfile.tmpl index b14a70ebe5..681cd3d4b6 100644 --- a/templates/workspaces/base/Dockerfile.tmpl +++ b/templates/workspaces/base/Dockerfile.tmpl @@ -23,10 +23,3 @@ RUN apt-get update && \ # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index 224188cd37..04bf439f46 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-base -version: 0.3.32 +version: 0.4.0 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/unrestricted/Dockerfile.tmpl b/templates/workspaces/unrestricted/Dockerfile.tmpl index e8dd45884d..0d40957347 100644 --- a/templates/workspaces/unrestricted/Dockerfile.tmpl +++ b/templates/workspaces/unrestricted/Dockerfile.tmpl @@ -31,11 +31,3 @@ RUN curl -o azuretre.tar.gz -L "https://github.com/microsoft/AzureTRE/archive/r # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR - -# Mirror plugins to prevent network access at runtime -# Remove when available from https://github.com/getporter/terraform-mixin/issues/90 -WORKDIR $BUNDLE_DIR/terraform -RUN terraform init -backend=false \ - && rm -fr $BUNDLE_DIR/terraform/.terraform/providers \ - && terraform providers mirror /usr/local/share/terraform/plugins - diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index d2363d8c28..539dbc5566 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-unrestricted -version: 0.1.9 +version: 0.2.0 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre From fb947ea1ee3bf749634370719a7a96e2e58d2369 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 12:14:05 +0000 Subject: [PATCH 09/16] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad4a6124a6..4ae7e65f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ ENHANCEMENTS: * Airlock requests with status "blocked_by_scan" have the reason for being blocked by the malware scanner in the status_message field ([#2666](https://github.com/microsoft/AzureTRE/pull/2666)) * Move admin-vm from core to a shared service ([#2624](https://github.com/microsoft/AzureTRE/pull/2624)) * Remove obsolete docker environment variables ([#2675](https://github.com/microsoft/AzureTRE/pull/2675)) +* Using Porter's Terrform mixin 1.0.0-rc.1 where mirror in done internally ([#2677](https://github.com/microsoft/AzureTRE/pull/2677)) BUG FIXES: From e29e6b61c0701d751ed753a13f2ad2844b0a7a3f Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 12:19:10 +0000 Subject: [PATCH 10/16] Airlock func private endpoint for internal storage --- .../terraform/airlock/airlock_processor.tf | 43 ++++++++++++++++--- .../terraform/airlock/storage_accounts.tf | 15 +++---- templates/core/terraform/airlock/variables.tf | 5 +++ templates/core/terraform/main.tf | 4 ++ templates/core/terraform/network/dns_zones.tf | 20 +++++++++ templates/core/terraform/network/locals.tf | 5 +++ templates/core/terraform/network/outputs.tf | 22 +++++++--- 7 files changed, 91 insertions(+), 23 deletions(-) diff --git a/templates/core/terraform/airlock/airlock_processor.tf b/templates/core/terraform/airlock/airlock_processor.tf index b7ec9bb62a..957ab8d9a5 100644 --- a/templates/core/terraform/airlock/airlock_processor.tf +++ b/templates/core/terraform/airlock/airlock_processor.tf @@ -31,13 +31,14 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" { } resource "azurerm_linux_function_app" "airlock_function_app" { - name = local.airlock_function_app_name - resource_group_name = var.resource_group_name - location = var.location - https_only = true - virtual_network_subnet_id = var.airlock_processor_subnet_id - service_plan_id = azurerm_service_plan.airlock_plan.id - storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name + name = local.airlock_function_app_name + resource_group_name = var.resource_group_name + location = var.location + https_only = true + virtual_network_subnet_id = var.airlock_processor_subnet_id + service_plan_id = azurerm_service_plan.airlock_plan.id + storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name + # consider moving to a managed identity here storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key tags = var.tre_core_tags @@ -112,3 +113,31 @@ resource "azurerm_monitor_diagnostic_setting" "airlock_function_app" { } } } + +resource "azurerm_private_endpoint" "function_storage" { + for_each = { + Blob = var.blob_core_dns_zone_id + File = var.file_core_dns_zone_id + Queue = var.queue_core_dns_zone_id + Table = var.table_core_dns_zone_id + } + name = "pe-${local.airlock_function_sa_name}-${lower(each.key)}" + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.airlock_storage_subnet_id + tags = var.tre_core_tags + + lifecycle { ignore_changes = [tags] } + + private_dns_zone_group { + name = "private-dns-zone-group-${local.airlock_function_sa_name}" + private_dns_zone_ids = [each.value] + } + + private_service_connection { + name = "psc-${local.airlock_function_sa_name}" + private_connection_resource_id = azurerm_storage_account.sa_import_in_progress.id + is_manual_connection = false + subresource_names = [each.key] + } +} diff --git a/templates/core/terraform/airlock/storage_accounts.tf b/templates/core/terraform/airlock/storage_accounts.tf index dbb4d3175b..08f9f37ed8 100644 --- a/templates/core/terraform/airlock/storage_accounts.tf +++ b/templates/core/terraform/airlock/storage_accounts.tf @@ -20,11 +20,6 @@ resource "azurerm_storage_account" "sa_import_external" { lifecycle { ignore_changes = [tags] } } -data "azurerm_private_dns_zone" "blobcore" { - name = "privatelink.blob.core.windows.net" - resource_group_name = var.resource_group_name -} - resource "azurerm_private_endpoint" "stg_import_external_pe" { name = "stg-ex-import-blob-${var.tre_id}" location = var.location @@ -36,7 +31,7 @@ resource "azurerm_private_endpoint" "stg_import_external_pe" { private_dns_zone_group { name = "private-dns-zone-group-stg-export-app" - private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id] + private_dns_zone_ids = [var.blob_core_dns_zone_id] } private_service_connection { @@ -80,7 +75,7 @@ resource "azurerm_private_endpoint" "stg_export_approved_pe" { private_dns_zone_group { name = "private-dns-zone-group-stg-export-app" - private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id] + private_dns_zone_ids = [var.blob_core_dns_zone_id] } private_service_connection { @@ -127,7 +122,7 @@ resource "azurerm_private_endpoint" "stg_import_inprogress_pe" { private_dns_zone_group { name = "private-dns-zone-group-stg-import-ip" - private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id] + private_dns_zone_ids = [var.blob_core_dns_zone_id] } private_service_connection { @@ -172,7 +167,7 @@ resource "azurerm_private_endpoint" "stg_import_rejected_pe" { private_dns_zone_group { name = "private-dns-zone-group-stg-import-rej" - private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id] + private_dns_zone_ids = [var.blob_core_dns_zone_id] } private_service_connection { @@ -220,7 +215,7 @@ resource "azurerm_private_endpoint" "stg_import_blocked_pe" { private_dns_zone_group { name = "private-dns-zone-group-stg-import-blocked" - private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id] + private_dns_zone_ids = [var.blob_core_dns_zone_id] } private_service_connection { diff --git a/templates/core/terraform/airlock/variables.tf b/templates/core/terraform/airlock/variables.tf index 79d95f8088..2b0526830a 100644 --- a/templates/core/terraform/airlock/variables.tf +++ b/templates/core/terraform/airlock/variables.tf @@ -51,3 +51,8 @@ variable "enable_malware_scanning" { } variable "log_analytics_workspace_id" {} + +variable "blob_core_dns_zone_id" {} +variable "file_core_dns_zone_id" {} +variable "queue_core_dns_zone_id" {} +variable "table_core_dns_zone_id" {} diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index 91f82b938b..b53204a5c3 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -116,6 +116,10 @@ module "airlock_resources" { enable_malware_scanning = var.enable_airlock_malware_scanning tre_core_tags = local.tre_core_tags log_analytics_workspace_id = module.azure_monitor.log_analytics_workspace_id + blob_core_dns_zone_id = module.network.blob_core_dns_zone_id + file_core_dns_zone_id = module.network.file_core_dns_zone_id + queue_core_dns_zone_id = module.network.queue_core_dns_zone_id + table_core_dns_zone_id = module.network.table_core_dns_zone_id enable_local_debugging = var.enable_local_debugging myip = local.myip diff --git a/templates/core/terraform/network/dns_zones.tf b/templates/core/terraform/network/dns_zones.tf index 497a977fc0..1684e45a9d 100644 --- a/templates/core/terraform/network/dns_zones.tf +++ b/templates/core/terraform/network/dns_zones.tf @@ -256,3 +256,23 @@ resource "azurerm_private_dns_zone_virtual_network_link" "eventgridlink" { lifecycle { ignore_changes = [tags] } } + +resource "azurerm_private_dns_zone" "private_dns_zones" { + for_each = local.private_dns_zone_names + name = each.key + resource_group_name = var.resource_group_name + tags = local.tre_core_tags + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_private_dns_zone_virtual_network_link" "private_dns_zone_links" { + for_each = azurerm_private_dns_zone.private_dns_zones + name = each.value.name + resource_group_name = var.resource_group_name + private_dns_zone_name = each.value.name + virtual_network_id = azurerm_virtual_network.core.id + tags = local.tre_core_tags + + lifecycle { ignore_changes = [tags] } +} diff --git a/templates/core/terraform/network/locals.tf b/templates/core/terraform/network/locals.tf index 97cdeb4ef1..286de4d67f 100644 --- a/templates/core/terraform/network/locals.tf +++ b/templates/core/terraform/network/locals.tf @@ -22,4 +22,9 @@ locals { tre_id = var.tre_id tre_core_service_id = var.tre_id } + + private_dns_zone_names = toset([ + "privatelink.queue.core.windows.net", + "privatelink.table.core.windows.net", + ]) } diff --git a/templates/core/terraform/network/outputs.tf b/templates/core/terraform/network/outputs.tf index f018fe75a9..752cfb0a05 100644 --- a/templates/core/terraform/network/outputs.tf +++ b/templates/core/terraform/network/outputs.tf @@ -34,18 +34,16 @@ output "airlock_events_subnet_id" { value = azurerm_subnet.airlock_events.id } -output "private_dns_zone_azurewebsites_id" { - value = azurerm_private_dns_zone.azurewebsites.id +output "resource_processor_subnet_id" { + value = azurerm_subnet.resource_processor.id } +# DNS Zones + output "private_dns_zone_mysql_id" { value = azurerm_private_dns_zone.mysql.id } -output "resource_processor_subnet_id" { - value = azurerm_subnet.resource_processor.id -} - output "azure_monitor_dns_zone_id" { value = azurerm_private_dns_zone.azure_monitor.id } @@ -73,3 +71,15 @@ output "azurewebsites_dns_zone_id" { output "static_web_dns_zone_id" { value = azurerm_private_dns_zone.static_web.id } + +output "file_core_dns_zone_id" { + value = azurerm_private_dns_zone.filecore.id +} + +output "queue_core_dns_zone_id" { + value = azurerm_private_dns_zone.private_dns_zones["privatelink.queue.core.windows.net"].id +} + +output "table_core_dns_zone_id" { + value = azurerm_private_dns_zone.private_dns_zones["privatelink.table.core.windows.net"].id +} From 86c28721fe5ebc94684b331c5aed7c366a97d58b Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 12:29:57 +0000 Subject: [PATCH 11/16] updates --- CHANGELOG.md | 1 + templates/core/terraform/airlock/airlock_processor.tf | 1 + templates/core/version.txt | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ae7e65f5c..0d14e73768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ ENHANCEMENTS: * Move admin-vm from core to a shared service ([#2624](https://github.com/microsoft/AzureTRE/pull/2624)) * Remove obsolete docker environment variables ([#2675](https://github.com/microsoft/AzureTRE/pull/2675)) * Using Porter's Terrform mixin 1.0.0-rc.1 where mirror in done internally ([#2677](https://github.com/microsoft/AzureTRE/pull/2677)) +* Airlock function internal storage is accessed with private endpoints ([#TBD](https://github.com/microsoft/AzureTRE/pull/TBD)) BUG FIXES: diff --git a/templates/core/terraform/airlock/airlock_processor.tf b/templates/core/terraform/airlock/airlock_processor.tf index 957ab8d9a5..e9d873af23 100644 --- a/templates/core/terraform/airlock/airlock_processor.tf +++ b/templates/core/terraform/airlock/airlock_processor.tf @@ -25,6 +25,7 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" { account_tier = "Standard" account_replication_type = "LRS" allow_nested_items_to_be_public = false + public_network_access_enabled = false tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } diff --git a/templates/core/version.txt b/templates/core/version.txt index b6f65f35da..e2b01a98c0 100644 --- a/templates/core/version.txt +++ b/templates/core/version.txt @@ -1 +1 @@ -__version__ = "0.4.30" +__version__ = "0.4.31" From 721fb72c2302ccf735cb9bba2fa3637b03117c0d Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 12:53:17 +0000 Subject: [PATCH 12/16] update azurerm provider --- templates/core/terraform/.terraform.lock.hcl | 28 ++++++++++---------- templates/core/terraform/main.tf | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 0244cb279e..932eb9f5c7 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -2,21 +2,21 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.19.1" - constraints = ">= 3.8.0, >= 3.16.0, 3.19.1" + version = "3.22.0" + constraints = ">= 3.8.0, >= 3.16.0, 3.22.0" hashes = [ - "h1:S50prXRxznk1Mk+SAUUsuDTGS96uxJMXVYHYlAYwYXk=", - "zh:0f3be13c20832d64f587e44e2aa461d6573d9ee4d870e9cfdb1f9e41a5f0ebcc", - "zh:17dcf38e9cc9b930c4f2f653ec71125e7ec5da72749f950234afb65a9d8bbb89", - "zh:2c8f949aec006afecd9286e9193f2894b036759fa8583e01e28eadd39b4a805b", - "zh:3a07630c200c09416dcfcbafdf535718cee54367c3b4dde6f7a92d98b43455b5", - "zh:422d8dc6a625927a04fc6c0f8e050ff92e3165e213b2db7ff1b6d78da0c2cb8b", - "zh:5bd44982a537906aea8dda96437dc4f98a39cc3e0b7ee52e87e1a7cda8d2ebf3", - "zh:91a75e8306947e58b58f0906f1540f3197426c993844cfc9504caa7aff9c62a4", - "zh:9e46d8baf67f97af34904de29b2c83c1fa7d1dc1f618f22c137ea15504f0562b", - "zh:bc13b3fced8df07c43b0a04fb3a88ae1b95ee9932b46b1a3d487f0d549ff9714", - "zh:bd10e9a68247951a7b5045b35f9058f1d11b8178c33e8323bf201f2339c9d0e2", - "zh:f41fe475fa54050fa3cae8b2cbb1bf350028acac4a129455cf018b5f7a666e00", + "h1:bxkMviG7vvNV2aPZQPall0mdIGOojsYeJvKbscPCZeM=", + "zh:03441438f73965fef3a60582573dc9137baf3142d157f16a8c187f7995bf968e", + "zh:1a45946e3ad479745e01eb28283beba4b7c63a94d29ccd3afa3adb8aac41ffa7", + "zh:457352525d3744a9f5d809a68e61ba51ad022fa012d0f092f04e31730700977d", + "zh:48c4ac83fbf5c7295ffe9b8f6a2f3e25d40361b53a8c77f1516973c714862805", + "zh:48c503892d780977405b4ef23db55d1216bbe96a592de63769f827cf3d5e092a", + "zh:5d5935681f91af8a44772262d7f6f1ed0a4b4e113236cc166559ff57b2c936c4", + "zh:61377b5edefdfe96b160a10b1b86b6faef02b813ea7d3d9cbcd8bc664c3293ed", + "zh:73b0696146afd6ff360138425973b3349cb2a45f13094a861d9c162c23e0d796", + "zh:8b2178ca3e1618107a7d5d68f57ca239c68b70a60cdae1c0a3e3ba867282ba25", + "zh:a4021c34ee777863f032425774485adab1d4aba10ce38eb415b5c3a3179423a4", + "zh:c66daaf59d5750b1e49706ffa052cb4467280b0cb481fdd4f7618bb8b9d1edb1", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index b53204a5c3..9b88109736 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "=3.19.1" + version = "=3.22.0" } random = { source = "hashicorp/random" From 8c93a6972cf1bf6b05e79c7b4d87017f9af3bc7f Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 13:01:32 +0000 Subject: [PATCH 13/16] remove unused var --- templates/core/terraform/airlock/variables.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/templates/core/terraform/airlock/variables.tf b/templates/core/terraform/airlock/variables.tf index 2b0526830a..adfa8f0a08 100644 --- a/templates/core/terraform/airlock/variables.tf +++ b/templates/core/terraform/airlock/variables.tf @@ -28,12 +28,6 @@ variable "mgmt_acr_name" { description = "Management ACR name" } -variable "arm_subscription_id" { - description = "The TRE subscription id." - type = string - default = "" -} - variable "airlock_app_service_plan_sku_size" { type = string default = "P1v3" From 1b609e99622b551e5c4b783b7df0ac8f82873637 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 16:03:49 +0000 Subject: [PATCH 14/16] fix --- templates/core/terraform/airlock/airlock_processor.tf | 2 +- templates/core/terraform/airlock/locals.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/core/terraform/airlock/airlock_processor.tf b/templates/core/terraform/airlock/airlock_processor.tf index e9d873af23..64c80c17b2 100644 --- a/templates/core/terraform/airlock/airlock_processor.tf +++ b/templates/core/terraform/airlock/airlock_processor.tf @@ -25,7 +25,6 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" { account_tier = "Standard" account_replication_type = "LRS" allow_nested_items_to_be_public = false - public_network_access_enabled = false tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } @@ -41,6 +40,7 @@ resource "azurerm_linux_function_app" "airlock_function_app" { storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name # consider moving to a managed identity here storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key + version = "~4" tags = var.tre_core_tags identity { diff --git a/templates/core/terraform/airlock/locals.tf b/templates/core/terraform/airlock/locals.tf index b288c50bfb..6a108f79a9 100644 --- a/templates/core/terraform/airlock/locals.tf +++ b/templates/core/terraform/airlock/locals.tf @@ -41,7 +41,7 @@ locals { export_approved_eventgrid_subscription_name = "evgs-airlock-export-approved-blob-created" airlock_function_app_name = "func-airlock-processor-${var.tre_id}" - airlock_function_sa_name = lower(replace("saairlockp${var.tre_id}", "-", "")) + airlock_function_sa_name = lower(replace("stairlockp${var.tre_id}", "-", "")) airlock_sa_blob_data_contributor = [ azurerm_storage_account.sa_import_external.id, From 43fa9dfb4cd9376edb7a55a3a07f1e5f75152829 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 16:06:08 +0000 Subject: [PATCH 15/16] reformat --- templates/core/terraform/airlock/airlock_processor.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/airlock/airlock_processor.tf b/templates/core/terraform/airlock/airlock_processor.tf index 64c80c17b2..de223199a1 100644 --- a/templates/core/terraform/airlock/airlock_processor.tf +++ b/templates/core/terraform/airlock/airlock_processor.tf @@ -40,7 +40,7 @@ resource "azurerm_linux_function_app" "airlock_function_app" { storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name # consider moving to a managed identity here storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key - version = "~4" + version = "~4" tags = var.tre_core_tags identity { From 21ae692c26b1facde0afd8ff26e500c26d71eef6 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 2 Oct 2022 16:11:11 +0000 Subject: [PATCH 16/16] fix --- templates/core/terraform/airlock/airlock_processor.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/core/terraform/airlock/airlock_processor.tf b/templates/core/terraform/airlock/airlock_processor.tf index de223199a1..957ab8d9a5 100644 --- a/templates/core/terraform/airlock/airlock_processor.tf +++ b/templates/core/terraform/airlock/airlock_processor.tf @@ -40,7 +40,6 @@ resource "azurerm_linux_function_app" "airlock_function_app" { storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name # consider moving to a managed identity here storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key - version = "~4" tags = var.tre_core_tags identity {