From 91ecce7f85e457e3af86aea772b06689cf6090d9 Mon Sep 17 00:00:00 2001
From: lolorol <lalesle@microsoft.com>
Date: Fri, 16 Apr 2021 11:23:04 +0000
Subject: [PATCH 1/7] fmt

---
 caf_launchpad/main.tf                         |  8 +++----
 .../azure_devops/locals.remote_tfstates.tf    |  8 +++----
 caf_solution/add-ons/azure_devops/main.tf     |  4 ++--
 caf_solution/add-ons/azure_devops/output.tf   |  2 +-
 .../locals.current_tfstates.tf                |  8 +++----
 .../add-ons/azure_devops_agent/main.tf        |  4 ++--
 .../add-ons/caf_eslz/custom_landing_zones.tf  |  2 +-
 caf_solution/add-ons/databricks/main.tf       |  6 ++---
 caf_solution/add-ons/terraform_cloud/main.tf  |  4 ++--
 caf_solution/landingzone.tf                   |  2 +-
 caf_solution/main.tf                          |  2 +-
 .../configuration.tfvars                      | 12 +++++-----
 .../201-multi-region-hub/configuration.tfvars | 24 +++++++++----------
 13 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/caf_launchpad/main.tf b/caf_launchpad/main.tf
index 76b3fbfe4..f2a158c8e 100644
--- a/caf_launchpad/main.tf
+++ b/caf_launchpad/main.tf
@@ -64,10 +64,10 @@ locals {
   }
 
   tfstates = tomap(
-      {
-        (var.landingzone.key) = local.backend[var.landingzone.backend_type]
-      }
-    )
+    {
+      (var.landingzone.key) = local.backend[var.landingzone.backend_type]
+    }
+  )
 
   backend = {
     azurerm = {
diff --git a/caf_solution/add-ons/azure_devops/locals.remote_tfstates.tf b/caf_solution/add-ons/azure_devops/locals.remote_tfstates.tf
index d5079932d..bbd533df1 100644
--- a/caf_solution/add-ons/azure_devops/locals.remote_tfstates.tf
+++ b/caf_solution/add-ons/azure_devops/locals.remote_tfstates.tf
@@ -37,10 +37,10 @@ locals {
   diagnostics     = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics
 
   combined = {
-    aad_apps           = merge(local.remote.aad_apps, tomap({(var.landingzone.key) = module.caf.aad_apps}))
-    azuread_groups     = merge(local.remote.azuread_groups, tomap({(var.landingzone.key) = module.caf.azuread_groups}))
-    keyvaults          = merge(local.remote.keyvaults, tomap({(var.landingzone.key) = module.caf.keyvaults}))
-    managed_identities = merge(local.remote.managed_identities, tomap({(var.landingzone.key) = module.caf.managed_identities}))
+    aad_apps           = merge(local.remote.aad_apps, tomap({ (var.landingzone.key) = module.caf.aad_apps }))
+    azuread_groups     = merge(local.remote.azuread_groups, tomap({ (var.landingzone.key) = module.caf.azuread_groups }))
+    keyvaults          = merge(local.remote.keyvaults, tomap({ (var.landingzone.key) = module.caf.keyvaults }))
+    managed_identities = merge(local.remote.managed_identities, tomap({ (var.landingzone.key) = module.caf.managed_identities }))
   }
 
   remote = {
diff --git a/caf_solution/add-ons/azure_devops/main.tf b/caf_solution/add-ons/azure_devops/main.tf
index 339350608..1cf69db04 100644
--- a/caf_solution/add-ons/azure_devops/main.tf
+++ b/caf_solution/add-ons/azure_devops/main.tf
@@ -53,14 +53,14 @@ locals {
   tfstates = merge(
     tomap(
       {
-        (var.landingzone.key) =local.backend[var.landingzone.backend_type]
+        (var.landingzone.key) = local.backend[var.landingzone.backend_type]
       }
     )
     ,
     data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates
   )
 
-  
+
   backend = {
     azurerm = {
       storage_account_name = var.tfstate_storage_account_name
diff --git a/caf_solution/add-ons/azure_devops/output.tf b/caf_solution/add-ons/azure_devops/output.tf
index ca68bc81c..c73344268 100644
--- a/caf_solution/add-ons/azure_devops/output.tf
+++ b/caf_solution/add-ons/azure_devops/output.tf
@@ -1,6 +1,6 @@
 output "keyvaults" {
   value = tomap(
-    {    
+    {
       (var.landingzone.key) = module.caf.keyvaults
     }
   )
diff --git a/caf_solution/add-ons/azure_devops_agent/locals.current_tfstates.tf b/caf_solution/add-ons/azure_devops_agent/locals.current_tfstates.tf
index 3434a0fec..583bb62b2 100644
--- a/caf_solution/add-ons/azure_devops_agent/locals.current_tfstates.tf
+++ b/caf_solution/add-ons/azure_devops_agent/locals.current_tfstates.tf
@@ -37,10 +37,10 @@ locals {
   diagnostics     = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics
 
   combined = {
-    aad_apps           = merge(local.remote.aad_apps, tomap({(var.landingzone.key) = module.caf.aad_apps}))
-    azuread_groups     = merge(local.remote.azuread_groups, tomap({(var.landingzone.key) = module.caf.azuread_groups}))
-    keyvaults          = merge(local.remote.keyvaults, tomap({(var.landingzone.key) = module.caf.keyvaults}))
-    managed_identities = merge(local.remote.managed_identities, tomap({(var.landingzone.key) = module.caf.managed_identities}))
+    aad_apps           = merge(local.remote.aad_apps, tomap({ (var.landingzone.key) = module.caf.aad_apps }))
+    azuread_groups     = merge(local.remote.azuread_groups, tomap({ (var.landingzone.key) = module.caf.azuread_groups }))
+    keyvaults          = merge(local.remote.keyvaults, tomap({ (var.landingzone.key) = module.caf.keyvaults }))
+    managed_identities = merge(local.remote.managed_identities, tomap({ (var.landingzone.key) = module.caf.managed_identities }))
   }
 
   remote = {
diff --git a/caf_solution/add-ons/azure_devops_agent/main.tf b/caf_solution/add-ons/azure_devops_agent/main.tf
index 2dc2779ab..38c9df491 100644
--- a/caf_solution/add-ons/azure_devops_agent/main.tf
+++ b/caf_solution/add-ons/azure_devops_agent/main.tf
@@ -53,14 +53,14 @@ locals {
   tfstates = merge(
     tomap(
       {
-        (var.landingzone.key) =local.backend[var.landingzone.backend_type]
+        (var.landingzone.key) = local.backend[var.landingzone.backend_type]
       }
     )
     ,
     data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates
   )
 
-  
+
   backend = {
     azurerm = {
       storage_account_name = var.tfstate_storage_account_name
diff --git a/caf_solution/add-ons/caf_eslz/custom_landing_zones.tf b/caf_solution/add-ons/caf_eslz/custom_landing_zones.tf
index f36f253b9..1d850eb80 100644
--- a/caf_solution/add-ons/caf_eslz/custom_landing_zones.tf
+++ b/caf_solution/add-ons/caf_eslz/custom_landing_zones.tf
@@ -25,7 +25,7 @@ locals {
               [
                 data.azurerm_management_group.id[mg_id].subscription_ids
               ]
-            ), 
+            ),
             []
           ),
           flatten(
diff --git a/caf_solution/add-ons/databricks/main.tf b/caf_solution/add-ons/databricks/main.tf
index 1007e8245..513475308 100644
--- a/caf_solution/add-ons/databricks/main.tf
+++ b/caf_solution/add-ons/databricks/main.tf
@@ -44,20 +44,20 @@ locals {
     log_analytics            = data.terraform_remote_state.landingzone.outputs.diagnostics.log_analytics
   }
 
-  
+
 
   # Update the tfstates map
   tfstates = merge(
     tomap(
       {
-        (var.landingzone.key) =local.backend[var.landingzone.backend_type]
+        (var.landingzone.key) = local.backend[var.landingzone.backend_type]
       }
     )
     ,
     data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates
   )
 
-  
+
   backend = {
     azurerm = {
       storage_account_name = var.tfstate_storage_account_name
diff --git a/caf_solution/add-ons/terraform_cloud/main.tf b/caf_solution/add-ons/terraform_cloud/main.tf
index 894b1304f..05c38ec9d 100644
--- a/caf_solution/add-ons/terraform_cloud/main.tf
+++ b/caf_solution/add-ons/terraform_cloud/main.tf
@@ -44,14 +44,14 @@ locals {
   tfstates = merge(
     tomap(
       {
-        (var.landingzone.key) =local.backend["tfc"]
+        (var.landingzone.key) = local.backend["tfc"]
       }
     )
     ,
     data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates
   )
 
-  
+
   backend = {
     tfc = {
       level           = var.landingzone.level,
diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf
index e4030f5e0..0d44997cb 100644
--- a/caf_solution/landingzone.tf
+++ b/caf_solution/landingzone.tf
@@ -1,7 +1,7 @@
 module "solution" {
   source  = "aztfmod/caf/azurerm"
   version = "~>5.3.0"
-  
+
   # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
 
 
diff --git a/caf_solution/main.tf b/caf_solution/main.tf
index 3eca96c6a..216cc29da 100644
--- a/caf_solution/main.tf
+++ b/caf_solution/main.tf
@@ -54,7 +54,7 @@ locals {
     data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates
   )
 
-  
+
   backend = {
     azurerm = {
       storage_account_name = var.tfstate_storage_account_name
diff --git a/caf_solution/scenario/networking/200-single-region-hub/configuration.tfvars b/caf_solution/scenario/networking/200-single-region-hub/configuration.tfvars
index 3e4945769..6a6126d07 100644
--- a/caf_solution/scenario/networking/200-single-region-hub/configuration.tfvars
+++ b/caf_solution/scenario/networking/200-single-region-hub/configuration.tfvars
@@ -75,9 +75,9 @@ vnet_peerings = {
       vnet_key = "hub_rg1"
     }
     to = {
-      lz_key      = "caf_gitops"
-      output_key  = "vnets"
-      vnet_key    = "devops_region1"
+      lz_key     = "caf_gitops"
+      output_key = "vnets"
+      vnet_key   = "devops_region1"
     }
     allow_virtual_network_access = true
     allow_forwarded_traffic      = false
@@ -89,9 +89,9 @@ vnet_peerings = {
   launchpad_devops-TO-hub_rg1 = {
     name = "launchpad_devops-TO-hub_rg1"
     from = {
-      lz_key      = "caf_gitops"
-      output_key  = "vnets"
-      vnet_key    = "devops_region1"
+      lz_key     = "caf_gitops"
+      output_key = "vnets"
+      vnet_key   = "devops_region1"
     }
     to = {
       vnet_key = "hub_rg1"
diff --git a/caf_solution/scenario/networking/201-multi-region-hub/configuration.tfvars b/caf_solution/scenario/networking/201-multi-region-hub/configuration.tfvars
index 45eb82b65..77169e540 100644
--- a/caf_solution/scenario/networking/201-multi-region-hub/configuration.tfvars
+++ b/caf_solution/scenario/networking/201-multi-region-hub/configuration.tfvars
@@ -154,9 +154,9 @@ vnet_peerings = {
       vnet_key = "hub_rg1"
     }
     to = {
-      lz_key      = "caf_gitops"
-      output_key  = "vnets"
-      vnet_key    = "devops_region1"
+      lz_key     = "caf_gitops"
+      output_key = "vnets"
+      vnet_key   = "devops_region1"
     }
     allow_virtual_network_access = true
     allow_forwarded_traffic      = false
@@ -168,9 +168,9 @@ vnet_peerings = {
   launchpad_devops-TO-hub_rg1 = {
     name = "launchpad_devops-TO-hub_rg1"
     from = {
-      lz_key      = "caf_gitops"
-      output_key  = "vnets"
-      vnet_key    = "devops_region1"
+      lz_key     = "caf_gitops"
+      output_key = "vnets"
+      vnet_key   = "devops_region1"
     }
     to = {
       vnet_key = "hub_rg1"
@@ -189,9 +189,9 @@ vnet_peerings = {
       vnet_key = "hub_rg2"
     }
     to = {
-      lz_key      = "caf_gitops"
-      output_key  = "vnets"
-      vnet_key    = "devops_region1"
+      lz_key     = "caf_gitops"
+      output_key = "vnets"
+      vnet_key   = "devops_region1"
     }
     allow_virtual_network_access = true
     allow_forwarded_traffic      = false
@@ -203,9 +203,9 @@ vnet_peerings = {
   launchpad_devops-TO-hub_rg2 = {
     name = "launchpad_devops-TO-hub_rg2"
     from = {
-      lz_key      = "caf_gitops"
-      output_key  = "vnets"
-      vnet_key    = "devops_region1"
+      lz_key     = "caf_gitops"
+      output_key = "vnets"
+      vnet_key   = "devops_region1"
     }
     to = {
       vnet_key = "hub_rg2"

From 36a5bd5b747b5e35e4f581e62e55b61196f150bf Mon Sep 17 00:00:00 2001
From: lolorol <LaurentLesle@users.noreply.github.com>
Date: Tue, 20 Apr 2021 10:26:26 +0800
Subject: [PATCH 2/7] Add missing keyvault_access_policies_azuread_apps

---
 caf_solution/landingzone.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf
index 59393dd48..19e4ae698 100644
--- a/caf_solution/landingzone.tf
+++ b/caf_solution/landingzone.tf
@@ -25,6 +25,7 @@ module "solution" {
   event_hubs                            = var.event_hubs
   global_settings                       = local.global_settings
   keyvault_access_policies              = var.keyvault_access_policies
+  keyvault_access_policies_azuread_apps = var.keyvault_access_policies_azuread_apps
   keyvault_certificate_issuers          = var.keyvault_certificate_issuers
   keyvaults                             = var.keyvaults
   log_analytics                         = var.log_analytics

From 19962ccadc146fbda6617829f2a70e418f597a14 Mon Sep 17 00:00:00 2001
From: lolorol <lalesle@microsoft.com>
Date: Tue, 20 Apr 2021 04:10:01 +0000
Subject: [PATCH 3/7] Update to support kubeconfig removed from tfstate

---
 .../add-ons/aad-pod-identity/providers.tf     | 26 +++++++++++--------
 .../add-ons/aks-secure-baseline/main.tf       |  2 +-
 .../add-ons/aks-secure-baseline/providers.tf  | 22 +++++++++++-----
 caf_solution/landingzone.tf                   |  6 ++---
 4 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/caf_solution/add-ons/aad-pod-identity/providers.tf b/caf_solution/add-ons/aad-pod-identity/providers.tf
index be2b3abb1..bc0831228 100644
--- a/caf_solution/add-ons/aad-pod-identity/providers.tf
+++ b/caf_solution/add-ons/aad-pod-identity/providers.tf
@@ -13,20 +13,24 @@ provider "kubernetes" {
   cluster_ca_certificate = local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate
 }
 
-provider "kustomization" {
-  kubeconfig_raw = local.k8sconfigs[var.aks_cluster_key].kube_admin_config_raw
-}
-
 locals {
   k8sconfigs = {
     for key, value in var.aks_clusters : key => {
-      kube_admin_config_raw  = local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config_raw
-      host                   = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.host : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.host
-      username               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.username : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.username
-      password               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.password : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.password
-      client_certificate     = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.client_certificate) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.client_certificate)
-      client_key             = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.client_key) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.client_key)
-      cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.cluster_ca_certificate) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.cluster_ca_certificate)
+      kube_admin_config_raw  = data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config_raw
+      host                   = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host
+      username               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username
+      password               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password
+      client_certificate     = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate)
+      client_key             = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key)
+      cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate)
     }
   }
+}
+
+# Get kubeconfig from AKS clusters
+data "azurerm_kubernetes_cluster" "kubeconfig" {
+  for_each = var.aks_clusters
+
+  name                = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name
+  resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name
 }
\ No newline at end of file
diff --git a/caf_solution/add-ons/aks-secure-baseline/main.tf b/caf_solution/add-ons/aks-secure-baseline/main.tf
index a846d85a0..3a17f5c90 100644
--- a/caf_solution/add-ons/aks-secure-baseline/main.tf
+++ b/caf_solution/add-ons/aks-secure-baseline/main.tf
@@ -10,7 +10,7 @@ terraform {
     }
     kustomization = {
       source  = "kbst/kustomization"
-      version = ">= 0.4.0"
+      version = ">= 0.5.0"
     }
   }
   required_version = ">= 0.13"
diff --git a/caf_solution/add-ons/aks-secure-baseline/providers.tf b/caf_solution/add-ons/aks-secure-baseline/providers.tf
index be2b3abb1..06e22a736 100644
--- a/caf_solution/add-ons/aks-secure-baseline/providers.tf
+++ b/caf_solution/add-ons/aks-secure-baseline/providers.tf
@@ -20,13 +20,21 @@ provider "kustomization" {
 locals {
   k8sconfigs = {
     for key, value in var.aks_clusters : key => {
-      kube_admin_config_raw  = local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config_raw
-      host                   = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.host : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.host
-      username               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.username : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.username
-      password               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.password : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.password
-      client_certificate     = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.client_certificate) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.client_certificate)
-      client_key             = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.client_key) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.client_key)
-      cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.cluster_ca_certificate) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.cluster_ca_certificate)
+      kube_admin_config_raw  = data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config_raw
+      host                   = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host
+      username               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username
+      password               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password
+      client_certificate     = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate)
+      client_key             = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key)
+      cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate)
     }
   }
+}
+
+# Get kubeconfig from AKS clusters
+data "azurerm_kubernetes_cluster" "kubeconfig" {
+  for_each = var.aks_clusters
+
+  name                = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name
+  resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name
 }
\ No newline at end of file
diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf
index c4d36103b..0d9e246c3 100644
--- a/caf_solution/landingzone.tf
+++ b/caf_solution/landingzone.tf
@@ -1,8 +1,8 @@
 module "solution" {
-  source  = "aztfmod/caf/azurerm"
-  version = "~>5.3.0"
+  # source  = "aztfmod/caf/azurerm"
+  # version = "~>5.3.0"
 
-  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
+  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=acr_password_output"
 
 
   azuread_api_permissions               = var.azuread_api_permissions

From f2571d3e2405da384f8efdf76898345e98382be0 Mon Sep 17 00:00:00 2001
From: lolorol <lalesle@microsoft.com>
Date: Tue, 20 Apr 2021 07:00:51 +0000
Subject: [PATCH 4/7] Update helm chart providers

---
 caf_solution/add-ons/helm-charts/providers.tf | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/caf_solution/add-ons/helm-charts/providers.tf b/caf_solution/add-ons/helm-charts/providers.tf
index 3beaf3ead..6af45c41c 100644
--- a/caf_solution/add-ons/helm-charts/providers.tf
+++ b/caf_solution/add-ons/helm-charts/providers.tf
@@ -27,13 +27,20 @@ provider "helm" {
 locals {
   k8sconfigs = {
     for key, value in var.aks_clusters : key => {
-      kube_admin_config_raw  = local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config_raw
-      host                   = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.host : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.host
-      username               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.username : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.username
-      password               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.password : local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.password
-      client_certificate     = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.client_certificate) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.client_certificate)
-      client_key             = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.client_key) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.client_key)
-      cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_admin_config.0.cluster_ca_certificate) : base64decode(local.remote.aks_clusters[value.lz_key][value.key].kube_config.0.cluster_ca_certificate)
+      host                   = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host
+      username               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username
+      password               = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password
+      client_certificate     = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate)
+      client_key             = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key)
+      cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate)
     }
   }
+}
+
+# Get kubeconfig from AKS clusters
+data "azurerm_kubernetes_cluster" "kubeconfig" {
+  for_each = var.aks_clusters
+
+  name                = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name
+  resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name
 }
\ No newline at end of file

From e552e6ed727f19e53e065478c27ef01d2d48ef0b Mon Sep 17 00:00:00 2001
From: lolorol <LaurentLesle@users.noreply.github.com>
Date: Tue, 20 Apr 2021 20:37:18 +0000
Subject: [PATCH 5/7] Update with caf module pre-release

---
 caf_launchpad/dynamic_secrets.tf | 7 ++++---
 caf_launchpad/landingzone.tf     | 7 ++++---
 caf_solution/dynamic_secrets.tf  | 7 ++++---
 caf_solution/landingzone.tf      | 2 +-
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/caf_launchpad/dynamic_secrets.tf b/caf_launchpad/dynamic_secrets.tf
index 709e4951d..f6500d4d7 100644
--- a/caf_launchpad/dynamic_secrets.tf
+++ b/caf_launchpad/dynamic_secrets.tf
@@ -1,8 +1,9 @@
 
 module "dynamic_keyvault_secrets" {
-  source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
-  version = "~>5.3.0"
-  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
+  # source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
+  # version = "~>5.3.0"
+
+  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
 
 
   for_each = try(var.dynamic_keyvault_secrets, {})
diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf
index 1fceae9e5..e89eff977 100644
--- a/caf_launchpad/landingzone.tf
+++ b/caf_launchpad/landingzone.tf
@@ -1,7 +1,8 @@
 module "launchpad" {
-  source  = "aztfmod/caf/azurerm"
-  version = "~>5.3.0"
-  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
+  # source  = "aztfmod/caf/azurerm"
+  # version = "~>5.3.0"
+
+  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
 
 
   azuread_api_permissions               = var.azuread_api_permissions
diff --git a/caf_solution/dynamic_secrets.tf b/caf_solution/dynamic_secrets.tf
index 1bed46d7c..440d7883d 100644
--- a/caf_solution/dynamic_secrets.tf
+++ b/caf_solution/dynamic_secrets.tf
@@ -1,7 +1,8 @@
 module "dynamic_keyvault_secrets" {
-  source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
-  version = "~>5.3.0"
-  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
+  # source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
+  # version = "~>5.3.0"
+  
+  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
 
   for_each = {
     for keyvault_key, secrets in try(var.dynamic_keyvault_secrets, {}) : keyvault_key => {
diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf
index 0d9e246c3..beaa86bd9 100644
--- a/caf_solution/landingzone.tf
+++ b/caf_solution/landingzone.tf
@@ -2,7 +2,7 @@ module "solution" {
   # source  = "aztfmod/caf/azurerm"
   # version = "~>5.3.0"
 
-  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=acr_password_output"
+  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
 
 
   azuread_api_permissions               = var.azuread_api_permissions

From b84d781d55d6281ce97cc58b35a99cab14d97613 Mon Sep 17 00:00:00 2001
From: lolorol <LaurentLesle@users.noreply.github.com>
Date: Tue, 20 Apr 2021 22:46:38 +0000
Subject: [PATCH 6/7] Update with registry version 5.3.2

---
 caf_launchpad/dynamic_secrets.tf | 7 +++----
 caf_launchpad/landingzone.tf     | 6 +++---
 caf_solution/dynamic_secrets.tf  | 6 +++---
 caf_solution/landingzone.tf      | 6 +++---
 4 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/caf_launchpad/dynamic_secrets.tf b/caf_launchpad/dynamic_secrets.tf
index f6500d4d7..1183aed6b 100644
--- a/caf_launchpad/dynamic_secrets.tf
+++ b/caf_launchpad/dynamic_secrets.tf
@@ -1,10 +1,9 @@
 
 module "dynamic_keyvault_secrets" {
-  # source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
-  # version = "~>5.3.0"
-
-  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
+  source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
+  version = "~>5.3.2"
 
+  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
 
   for_each = try(var.dynamic_keyvault_secrets, {})
 
diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf
index e89eff977..915158c21 100644
--- a/caf_launchpad/landingzone.tf
+++ b/caf_launchpad/landingzone.tf
@@ -1,8 +1,8 @@
 module "launchpad" {
-  # source  = "aztfmod/caf/azurerm"
-  # version = "~>5.3.0"
+  source  = "aztfmod/caf/azurerm"
+  version = "~>5.3.2"
 
-  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
+  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
 
 
   azuread_api_permissions               = var.azuread_api_permissions
diff --git a/caf_solution/dynamic_secrets.tf b/caf_solution/dynamic_secrets.tf
index 440d7883d..8a890701f 100644
--- a/caf_solution/dynamic_secrets.tf
+++ b/caf_solution/dynamic_secrets.tf
@@ -1,8 +1,8 @@
 module "dynamic_keyvault_secrets" {
-  # source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
-  # version = "~>5.3.0"
+  source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
+  version = "~>5.3.2"
   
-  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
+  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
 
   for_each = {
     for keyvault_key, secrets in try(var.dynamic_keyvault_secrets, {}) : keyvault_key => {
diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf
index beaa86bd9..65c64c65e 100644
--- a/caf_solution/landingzone.tf
+++ b/caf_solution/landingzone.tf
@@ -1,8 +1,8 @@
 module "solution" {
-  # source  = "aztfmod/caf/azurerm"
-  # version = "~>5.3.0"
+  source  = "aztfmod/caf/azurerm"
+  version = "~>5.3.2"
 
-  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
+  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master"
 
 
   azuread_api_permissions               = var.azuread_api_permissions

From 8af8af4e41086db3baeb95df9fd255da1970bdfb Mon Sep 17 00:00:00 2001
From: Arnaud Lheureux <arnaud@microsoft.com>
Date: Wed, 21 Apr 2021 06:56:29 +0000
Subject: [PATCH 7/7] Update AzDO LZ

---
 caf_solution/add-ons/azure_devops/main.tf                  | 4 ----
 caf_solution/add-ons/azure_devops_agent/dynamic_secrets.tf | 6 +++---
 caf_solution/add-ons/azure_devops_agent/main.tf            | 4 ----
 caf_solution/add-ons/azure_devops_agent/solution.tf        | 2 +-
 4 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/caf_solution/add-ons/azure_devops/main.tf b/caf_solution/add-ons/azure_devops/main.tf
index 1cf69db04..e81716efc 100644
--- a/caf_solution/add-ons/azure_devops/main.tf
+++ b/caf_solution/add-ons/azure_devops/main.tf
@@ -1,9 +1,5 @@
 terraform {
   required_providers {
-    azurerm = {
-      source  = "hashicorp/azurerm"
-      version = "~> 2.55.0"
-    }
     azuread = {
       source  = "hashicorp/azuread"
       version = "~> 1.4.0"
diff --git a/caf_solution/add-ons/azure_devops_agent/dynamic_secrets.tf b/caf_solution/add-ons/azure_devops_agent/dynamic_secrets.tf
index ea18c3d5c..81f9f7249 100644
--- a/caf_solution/add-ons/azure_devops_agent/dynamic_secrets.tf
+++ b/caf_solution/add-ons/azure_devops_agent/dynamic_secrets.tf
@@ -1,8 +1,8 @@
 
 module "dynamic_keyvault_secrets" {
-  # source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
-  # version = "~>5.3.0"
-  source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
+  source  = "aztfmod/caf/azurerm//modules/security/dynamic_keyvault_secrets"
+  version = "~>5.3.0"
+  # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git//modules/security/dynamic_keyvault_secrets?ref=master"
 
 
   for_each = try(var.dynamic_keyvault_secrets, {})
diff --git a/caf_solution/add-ons/azure_devops_agent/main.tf b/caf_solution/add-ons/azure_devops_agent/main.tf
index 38c9df491..9fb1f0611 100644
--- a/caf_solution/add-ons/azure_devops_agent/main.tf
+++ b/caf_solution/add-ons/azure_devops_agent/main.tf
@@ -1,9 +1,5 @@
 terraform {
   required_providers {
-    azurerm = {
-      source  = "hashicorp/azurerm"
-      version = "~> 2.55"
-    }
     azuread = {
       source  = "hashicorp/azuread"
       version = "~> 1.4.0"
diff --git a/caf_solution/add-ons/azure_devops_agent/solution.tf b/caf_solution/add-ons/azure_devops_agent/solution.tf
index 3316f9894..6d87d2f5d 100644
--- a/caf_solution/add-ons/azure_devops_agent/solution.tf
+++ b/caf_solution/add-ons/azure_devops_agent/solution.tf
@@ -1,6 +1,6 @@
 module "caf" {
   source  = "aztfmod/caf/azurerm"
-  version = "~>5.1.0"
+  version = "~>5.3.0"
 
   current_landingzone_key     = var.landingzone.key
   tenant_id                   = var.tenant_id