From 58c1023d72d8d944747b9c85cc3090eecbeab220 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 24 Nov 2021 10:58:36 +0000 Subject: [PATCH 01/15] Add tenant ID as input variable --- terraform/azure/variables.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/terraform/azure/variables.tf b/terraform/azure/variables.tf index e8924440d2..a3e3fba68b 100644 --- a/terraform/azure/variables.tf +++ b/terraform/azure/variables.tf @@ -9,6 +9,16 @@ variable "subscription_id" { EOT } +variable "tenant_id" { + type = string + description = <<-EOT + Tenant ID inside which our subscription is housed + + `az account show -s SUBSCRIPTION_ID -o table` will show the ID of the tenant + after you have logged in with `az login`. + EOT +} + variable "resourcegroup_name" { type = string description = <<-EOT From 8aff34c3e50f070c31c11cf63998bd3f70e170d3 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 24 Nov 2021 10:58:52 +0000 Subject: [PATCH 02/15] Pin Azure RM and AD providers --- terraform/azure/main.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 6c0523aa03..2841b5b0b9 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -1,10 +1,25 @@ terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "2.86.0" + } + + azuread = { + source = "hashicorp/azuread" + version = "2.10.0" + } + } backend "gcs" { bucket = "two-eye-two-see-org-terraform-state" prefix = "terraform/state/pilot-hubs" } } + +provider "azuread" { + tenant_id = var.tenant_id +} provider "azurerm" { subscription_id = var.subscription_id features {} From 699e5f0cd2479a95bcedb331c6c07cbc5ed3c8fa Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 10:37:37 +0000 Subject: [PATCH 03/15] Add tenant ID to carbonplan's tfvars file --- terraform/azure/projects/carbonplan.tfvars | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/azure/projects/carbonplan.tfvars b/terraform/azure/projects/carbonplan.tfvars index d6a16fb338..aa091503c2 100644 --- a/terraform/azure/projects/carbonplan.tfvars +++ b/terraform/azure/projects/carbonplan.tfvars @@ -1,3 +1,4 @@ +tenant_id = "d6b0296c-4d43-4983-93b0-36248aa9c592" subscription_id = "c5e7a734-3dbf-4285-80e5-4c0afb1f65dc" resourcegroup_name = "2i2c-carbonplan-cluster" From 8f77a5ee2b6b5a7bf88ba98f6237721fdad4d1d5 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 10:37:58 +0000 Subject: [PATCH 04/15] First draft of service principal creation --- terraform/azure/service-principal.tf | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 terraform/azure/service-principal.tf diff --git a/terraform/azure/service-principal.tf b/terraform/azure/service-principal.tf new file mode 100644 index 0000000000..c9f2241771 --- /dev/null +++ b/terraform/azure/service-principal.tf @@ -0,0 +1,9 @@ +resource "azuread_service_principal" "service_principal" { + application_id = var.subscription_id + app_role_assignment_required = false + use_existing = true +} + +resource "azuread_service_principal_password" "service_principal_password" { + service_principal_id = azuread_service_principal.service_principal.object_id +} From 7d47ec7aa8dd5aea72a416244d5e0d89410e085b Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 11:05:53 +0000 Subject: [PATCH 05/15] Output service principal so it can be used to login --- terraform/azure/service-principal.tf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/terraform/azure/service-principal.tf b/terraform/azure/service-principal.tf index c9f2241771..ae3bb87a77 100644 --- a/terraform/azure/service-principal.tf +++ b/terraform/azure/service-principal.tf @@ -7,3 +7,17 @@ resource "azuread_service_principal" "service_principal" { resource "azuread_service_principal_password" "service_principal_password" { service_principal_id = azuread_service_principal.service_principal.object_id } + +locals{ + service_principal = { + "tenant_id": var.tenant_id, + "subscription_id": var.subscription_id, + "service_principal_id": azuread_service_principal.service_principal.object_id, + "service_principal_password": azuread_service_principal_password.service_principal_password.value + } +} + +output "service_principal_config" { + value = jsonencode(local.service_principal) + sensitive = true +} From d1f4966c2359094e5a53324f152fc24358da52dc Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 11:12:34 +0000 Subject: [PATCH 06/15] Link the service principal to the Kubernetes Cluster --- terraform/azure/main.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index b336f902da..7ae8fb9266 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -98,6 +98,11 @@ resource "azurerm_kubernetes_cluster" "jupyterhub" { network_plugin = "kubenet" network_policy = "calico" } + + service_principal { + client_id = azuread_service_principal.service_principal.object_id + client_secret = azuread_service_principal_password.service_principal_password.value + } } From ae265a2857e95fcb8b78bf6f25fd4c918d0ac734 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 11:14:23 +0000 Subject: [PATCH 07/15] Remove raw kubeconfig output We will be authenticating using the service principal instead --- terraform/azure/main.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 7ae8fb9266..5b329e2b8d 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -183,12 +183,6 @@ locals { } } - -output "kubeconfig" { - value = azurerm_kubernetes_cluster.jupyterhub.kube_config_raw - sensitive = true -} - output "registry_creds_config" { value = jsonencode(local.registry_creds) sensitive = true From 2b3e314f2ea9886252a820fda9937e5725759112 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 11:51:12 +0000 Subject: [PATCH 08/15] Revert "Remove raw kubeconfig output" This reverts commit ae265a2857e95fcb8b78bf6f25fd4c918d0ac734. --- terraform/azure/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 5b329e2b8d..7ae8fb9266 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -183,6 +183,12 @@ locals { } } + +output "kubeconfig" { + value = azurerm_kubernetes_cluster.jupyterhub.kube_config_raw + sensitive = true +} + output "registry_creds_config" { value = jsonencode(local.registry_creds) sensitive = true From c5aeaa3506d5bdbfbba0959e55ed07d3e3e67a0c Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 13:12:44 +0000 Subject: [PATCH 09/15] Add bool variable to optionally create service principal --- terraform/azure/variables.tf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/terraform/azure/variables.tf b/terraform/azure/variables.tf index a3e3fba68b..dceeed9104 100644 --- a/terraform/azure/variables.tf +++ b/terraform/azure/variables.tf @@ -104,3 +104,14 @@ variable "dask_nodes" { description = "Dask node pools to create. Defaults to notebook_nodes" default = {} } + +variable "create_service_principal" { + type = bool + default = false + description = <<-EOT + When true, create a Service Principal to authenticate with. + + This is a temporary fix to allow for the fact that we cannot create Service + Principals for UToronto. + EOT +} From d89f506e6270ad21f5ba03bd9c00f4cd0ba5496d Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 13:17:46 +0000 Subject: [PATCH 10/15] Use count parameter to optionally create a service principal --- terraform/azure/service-principal.tf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/terraform/azure/service-principal.tf b/terraform/azure/service-principal.tf index ae3bb87a77..4fe714a04f 100644 --- a/terraform/azure/service-principal.tf +++ b/terraform/azure/service-principal.tf @@ -1,19 +1,23 @@ resource "azuread_service_principal" "service_principal" { + count = var.create_service_principal ? 1 : 0 + application_id = var.subscription_id app_role_assignment_required = false use_existing = true } resource "azuread_service_principal_password" "service_principal_password" { - service_principal_id = azuread_service_principal.service_principal.object_id + count = var.create_service_principal ? 1 : 0 + + service_principal_id = azuread_service_principal.service_principal[1].object_id } locals{ service_principal = { "tenant_id": var.tenant_id, "subscription_id": var.subscription_id, - "service_principal_id": azuread_service_principal.service_principal.object_id, - "service_principal_password": azuread_service_principal_password.service_principal_password.value + "service_principal_id": var.create_service_principal ? azuread_service_principal.service_principal[1].object_id : "", + "service_principal_password": var.create_service_principal ? azuread_service_principal_password.service_principal_password[1].value : "" } } From 6eaea044fac07fe4489a1ac540a8e34b2dc1662a Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 13:18:14 +0000 Subject: [PATCH 11/15] Dynamically assign the service principal to the k8s cluster --- terraform/azure/main.tf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 7ae8fb9266..84878b741c 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -99,9 +99,13 @@ resource "azurerm_kubernetes_cluster" "jupyterhub" { network_policy = "calico" } - service_principal { - client_id = azuread_service_principal.service_principal.object_id - client_secret = azuread_service_principal_password.service_principal_password.value + dynamic "service_principal" { + for_each = var.create_service_principal ? [1] : [] + + content { + client_id = azuread_service_principal.service_principal[1].object_id + client_secret = azuread_service_principal_password.service_principal_password[1].value + } } } From dd9975f6928c7df7b5dcda84754107dc61bce802 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 13:49:01 +0000 Subject: [PATCH 12/15] Update incorrect docstring for dask_nodes variable --- terraform/azure/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/azure/variables.tf b/terraform/azure/variables.tf index dceeed9104..a8c43be841 100644 --- a/terraform/azure/variables.tf +++ b/terraform/azure/variables.tf @@ -101,7 +101,7 @@ variable "notebook_nodes" { variable "dask_nodes" { type = map(map(string)) - description = "Dask node pools to create. Defaults to notebook_nodes" + description = "Dask node pools to create" default = {} } From 84e5586aff941b82ffe891e5bdd1aba1ca1f1144 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 13:58:12 +0000 Subject: [PATCH 13/15] Fix and index error --- terraform/azure/main.tf | 4 ++-- terraform/azure/service-principal.tf | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 84878b741c..b2adb3c3bd 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -103,8 +103,8 @@ resource "azurerm_kubernetes_cluster" "jupyterhub" { for_each = var.create_service_principal ? [1] : [] content { - client_id = azuread_service_principal.service_principal[1].object_id - client_secret = azuread_service_principal_password.service_principal_password[1].value + client_id = azuread_service_principal.service_principal[0].object_id + client_secret = azuread_service_principal_password.service_principal_password[0].value } } } diff --git a/terraform/azure/service-principal.tf b/terraform/azure/service-principal.tf index 4fe714a04f..0a6d0fdc50 100644 --- a/terraform/azure/service-principal.tf +++ b/terraform/azure/service-principal.tf @@ -9,15 +9,15 @@ resource "azuread_service_principal" "service_principal" { resource "azuread_service_principal_password" "service_principal_password" { count = var.create_service_principal ? 1 : 0 - service_principal_id = azuread_service_principal.service_principal[1].object_id + service_principal_id = azuread_service_principal.service_principal[0].object_id } locals{ service_principal = { "tenant_id": var.tenant_id, "subscription_id": var.subscription_id, - "service_principal_id": var.create_service_principal ? azuread_service_principal.service_principal[1].object_id : "", - "service_principal_password": var.create_service_principal ? azuread_service_principal_password.service_principal_password[1].value : "" + "service_principal_id": var.create_service_principal ? azuread_service_principal.service_principal[0].object_id : "", + "service_principal_password": var.create_service_principal ? azuread_service_principal_password.service_principal_password[0].value : "" } } From 0c63e64b23e4368a3102430d9dd068c8757327ba Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Wed, 1 Dec 2021 13:58:30 +0000 Subject: [PATCH 14/15] Enable service principal create for carbonplan --- terraform/azure/projects/carbonplan.tfvars | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/azure/projects/carbonplan.tfvars b/terraform/azure/projects/carbonplan.tfvars index aa091503c2..aa33c3ac2f 100644 --- a/terraform/azure/projects/carbonplan.tfvars +++ b/terraform/azure/projects/carbonplan.tfvars @@ -1,6 +1,7 @@ tenant_id = "d6b0296c-4d43-4983-93b0-36248aa9c592" subscription_id = "c5e7a734-3dbf-4285-80e5-4c0afb1f65dc" resourcegroup_name = "2i2c-carbonplan-cluster" +create_service_principal = true ssh_pub_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrtD6l/2S2dDT1OFLIUa46KAuzlUkckneQ4oyK5c2izT0nlBOxOcWACvnJ4rqntZVuhML7GYS35tQQnXyEXctTS2LnB7illl+oCT63WQzhhutfPLZSW3gJ+/CFVVp0VdHq4t1O8CQO+EgeV1cvuBre/neNZo5tcqXEsoV7tw/qw8XmD7Dk3yPz4NEFYZyZ4xQM7Qn2j9krZlRO2wCrosMFLo5Rv108PSDTHn3VtXEpJwemUO93D0ipJvCDE62M3vJxamqP4k5HLVn3Jun5KEvA7fINvgP3NSXkPVEEE4Prcqc1IJIzSxzygEN7xkZYsUEfJkybGXUWtUXdNjOEfLLlhYToMhdcP4jZluAjTtQmnJTr1sLegH+kbGIvI8MFIJAgXXLXS8/ubXHMozb+4gXRhyT8SMPwmgXGA+FtDduy5gnpPMSgmozQY17av5r25T6viWX5ivtL+n/CNJGf4npM1vKWcUNKiGyJrwzxQdpwJZ2uMP3EIdCYxOvqCQ+7UEM= sgibson@Athena.broadband" From a35b84e50b76ec0cc250ae97f271d71ea6e4e4d3 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 2 Dec 2021 09:49:56 +0000 Subject: [PATCH 15/15] Revert changes to carbonplan.tfvars --- terraform/azure/projects/carbonplan.tfvars | 2 -- 1 file changed, 2 deletions(-) diff --git a/terraform/azure/projects/carbonplan.tfvars b/terraform/azure/projects/carbonplan.tfvars index aa33c3ac2f..d6a16fb338 100644 --- a/terraform/azure/projects/carbonplan.tfvars +++ b/terraform/azure/projects/carbonplan.tfvars @@ -1,7 +1,5 @@ -tenant_id = "d6b0296c-4d43-4983-93b0-36248aa9c592" subscription_id = "c5e7a734-3dbf-4285-80e5-4c0afb1f65dc" resourcegroup_name = "2i2c-carbonplan-cluster" -create_service_principal = true ssh_pub_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrtD6l/2S2dDT1OFLIUa46KAuzlUkckneQ4oyK5c2izT0nlBOxOcWACvnJ4rqntZVuhML7GYS35tQQnXyEXctTS2LnB7illl+oCT63WQzhhutfPLZSW3gJ+/CFVVp0VdHq4t1O8CQO+EgeV1cvuBre/neNZo5tcqXEsoV7tw/qw8XmD7Dk3yPz4NEFYZyZ4xQM7Qn2j9krZlRO2wCrosMFLo5Rv108PSDTHn3VtXEpJwemUO93D0ipJvCDE62M3vJxamqP4k5HLVn3Jun5KEvA7fINvgP3NSXkPVEEE4Prcqc1IJIzSxzygEN7xkZYsUEfJkybGXUWtUXdNjOEfLLlhYToMhdcP4jZluAjTtQmnJTr1sLegH+kbGIvI8MFIJAgXXLXS8/ubXHMozb+4gXRhyT8SMPwmgXGA+FtDduy5gnpPMSgmozQY17av5r25T6viWX5ivtL+n/CNJGf4npM1vKWcUNKiGyJrwzxQdpwJZ2uMP3EIdCYxOvqCQ+7UEM= sgibson@Athena.broadband"