Workload identity (#2460)

pyproject.toml

@@ -136,21 +136,23 @@ module = [
 ignore_missing_imports = true
 
 [tool.ruff]
+extend-exclude = [
+    "src/_nebari/template",
+    "home",
+    "__pycache__"
+]
+
+[tool.ruff.lint]
 select = [
-    "E",
-    "F",
-    "PTH",
+    "E",  # E: pycodestyle rules
+    "F",  # F: pyflakes rules
+    "PTH",  # PTH: flake8-use-pathlib rules
 ]
 ignore = [
     "E501", # Line too long
     "F821", # Undefined name
     "PTH123", # open() should be replaced by Path.open()
 ]
-extend-exclude = [
-    "src/_nebari/template",
-    "home",
-    "__pycache__"
-]
 
 [tool.coverage.run]
 branch = true

src/_nebari/stages/infrastructure/__init__.py

@@ -112,6 +112,7 @@ class AzureInputVars(schema.Base):
     tags: Dict[str, str] = {}
     max_pods: Optional[int] = None
     network_profile: Optional[Dict[str, str]] = None
+    workload_identity_enabled: bool = False
 
 
 class AWSNodeGroupInputVars(schema.Base):
@@ -380,6 +381,7 @@ class AzureProvider(schema.Base):
     tags: Optional[Dict[str, str]] = {}
     network_profile: Optional[Dict[str, str]] = None
     max_pods: Optional[int] = None
+    workload_identity_enabled: bool = False
 
     @model_validator(mode="before")
     @classmethod
@@ -781,6 +783,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 tags=self.config.azure.tags,
                 network_profile=self.config.azure.network_profile,
                 max_pods=self.config.azure.max_pods,
+                workload_identity_enabled=self.config.azure.workload_identity_enabled,
             ).model_dump()
         elif self.config.provider == schema.ProviderEnum.aws:
             return AWSInputVars(

src/_nebari/stages/infrastructure/template/azure/main.tf

@@ -40,6 +40,7 @@ module "kubernetes" {
       max_size      = config.max_nodes
     }
   ]
-  vnet_subnet_id          = var.vnet_subnet_id
-  private_cluster_enabled = var.private_cluster_enabled
+  vnet_subnet_id            = var.vnet_subnet_id
+  private_cluster_enabled   = var.private_cluster_enabled
+  workload_identity_enabled = var.workload_identity_enabled
 }

src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/main.tf

@@ -5,6 +5,10 @@ resource "azurerm_kubernetes_cluster" "main" {
   resource_group_name = var.resource_group_name
   tags                = var.tags
 
+  # To enable Azure AD Workload Identity oidc_issuer_enabled must be set to true.
+  oidc_issuer_enabled       = var.workload_identity_enabled
+  workload_identity_enabled = var.workload_identity_enabled
+
   # DNS prefix specified when creating the managed cluster. Changing this forces a new resource to be created.
   dns_prefix = "Nebari" # required
 
@@ -39,6 +43,9 @@ resource "azurerm_kubernetes_cluster" "main" {
       "azure-node-pool" = var.node_groups[0].name
     }
     tags = var.tags
+
+    # temparory_name_for_rotation must be <= 12 characters
+    temporary_name_for_rotation = "${substr(var.node_groups[0].name, 0, 9)}tmp"
   }
 
   sku_tier = "Free" # "Free" [Default] or "Paid"

src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/outputs.tf

@@ -17,3 +17,13 @@ output "kubeconfig" {
   sensitive   = true
   value       = azurerm_kubernetes_cluster.main.kube_config_raw
 }
+
+output "cluster_oidc_issuer_url" {
+  description = "The OpenID Connect issuer URL that is associated with the AKS cluster"
+  value       = azurerm_kubernetes_cluster.main.oidc_issuer_url
+}
+
+output "resource_group_name" {
+  description = "The name of the resource group in which the AKS cluster is created"
+  value       = azurerm_kubernetes_cluster.main.resource_group_name
+}

src/_nebari/stages/infrastructure/template/azure/modules/kubernetes/variables.tf

@@ -70,3 +70,9 @@ variable "max_pods" {
   type        = number
   default     = 60
 }
+
+variable "workload_identity_enabled" {
+  description = "Enable Workload Identity"
+  type        = bool
+  default     = false
+}

src/_nebari/stages/infrastructure/template/azure/outputs.tf

@@ -22,3 +22,13 @@ output "kubeconfig_filename" {
   description = "filename for nebari kubeconfig"
   value       = var.kubeconfig_filename
 }
+
+output "cluster_oidc_issuer_url" {
+  description = "The OpenID Connect issuer URL that is associated with the AKS cluster"
+  value       = module.kubernetes.cluster_oidc_issuer_url
+}
+
+output "resource_group_name" {
+  description = "The name of the resource group in which the AKS cluster is created"
+  value       = module.kubernetes.resource_group_name
+}

src/_nebari/stages/infrastructure/template/azure/variables.tf

@@ -76,3 +76,9 @@ variable "max_pods" {
   type        = number
   default     = 60
 }
+
+variable "workload_identity_enabled" {
+  description = "Enable Workload Identity"
+  type        = bool
+  default     = false
+}

src/_nebari/stages/kubernetes_services/__init__.py

@@ -24,6 +24,9 @@
 TIMEOUT = 10  # seconds
 
 
+_forwardauth_middleware_name = "traefik-forward-auth"
+
+
 @schema.yaml_object(schema.yaml)
 class AccessEnum(str, enum.Enum):
     all = "all"
@@ -327,6 +330,7 @@ class KubernetesServicesInputVars(schema.Base):
     realm_id: str
     node_groups: Dict[str, Dict[str, str]]
     jupyterhub_logout_redirect_url: str = Field(alias="jupyterhub-logout-redirect-url")
+    forwardauth_middleware_name: str = _forwardauth_middleware_name
 
 
 def _split_docker_image_name(image_name):
@@ -383,6 +387,7 @@ class DaskGatewayInputVars(schema.Base):
     dask_worker_image: ImageNameTag = Field(alias="dask-worker-image")
     dask_gateway_profiles: Dict[str, Any] = Field(alias="dask-gateway-profiles")
     cloud_provider: str = Field(alias="cloud-provider")
+    forwardauth_middleware_name: str = _forwardauth_middleware_name
 
 
 class MonitoringInputVars(schema.Base):

src/_nebari/stages/kubernetes_services/template/dask_gateway.tf

@@ -40,4 +40,6 @@ module "dask-gateway" {
   profiles = var.dask-gateway-profiles
 
   cloud-provider = var.cloud-provider
+
+  forwardauth_middleware_name = var.forwardauth_middleware_name
 }

src/_nebari/stages/kubernetes_services/template/forward-auth.tf

@@ -5,5 +5,21 @@ module "forwardauth" {
   external-url = var.endpoint
   realm_id     = var.realm_id
 
-  node-group = var.node_groups.general
+  node-group                  = var.node_groups.general
+  forwardauth_middleware_name = var.forwardauth_middleware_name
+}
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}
+
+output "forward-auth-middleware" {
+  description = "middleware name for use with forward auth"
+  value       = module.forwardauth.forward-auth-middleware
+}
+
+output "forward-auth-service" {
+  description = "middleware name for use with forward auth"
+  value       = module.forwardauth.forward-auth-service
 }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/main.tf

@@ -144,12 +144,12 @@ resource "kubernetes_manifest" "forwardauth-middleware" {
     apiVersion = "traefik.containo.us/v1alpha1"
     kind       = "Middleware"
     metadata = {
-      name      = "traefik-forward-auth"
+      name      = var.forwardauth_middleware_name
       namespace = var.namespace
     }
     spec = {
       forwardAuth = {
-        address = "http://forwardauth-service:4181"
+        address = "http://${kubernetes_service.forwardauth-service.metadata.0.name}:4181"
         authResponseHeaders = [
           "X-Forwarded-User"
         ]
@@ -175,7 +175,7 @@ resource "kubernetes_manifest" "forwardauth-ingressroute" {
 
           middlewares = [
             {
-              name      = "traefik-forward-auth"
+              name      = kubernetes_manifest.forwardauth-middleware.manifest.metadata.name
               namespace = var.namespace
             }
           ]

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/outputs.tf

@@ -0,0 +1,13 @@
+output "forward-auth-middleware" {
+  description = "middleware name for use with forward auth"
+  value = {
+    name = kubernetes_manifest.forwardauth-middleware.manifest.metadata.name
+  }
+}
+
+output "forward-auth-service" {
+  description = "middleware name for use with forward auth"
+  value = {
+    name = kubernetes_service.forwardauth-service.metadata.0.name
+  }
+}

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/forwardauth/variables.tf

@@ -26,3 +26,8 @@ variable "node-group" {
     value = string
   })
 }
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/middleware.tf

@@ -32,7 +32,7 @@ resource "kubernetes_manifest" "chain-middleware" {
       chain = {
         middlewares = [
           {
-            name      = "traefik-forward-auth"
+            name      = var.forwardauth_middleware_name
             namespace = var.namespace
           },
           {

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/variables.tf

@@ -204,3 +204,7 @@ variable "cloud-provider" {
   description = "Name of the cloud provider to deploy to."
   type        = string
 }
+
+variable "forwardauth_middleware_name" {
+  type = string
+}

src/_nebari/stages/nebari_tf_extensions/__init__.py

@@ -72,6 +72,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 "stages/05-kubernetes-keycloak"
             ]["keycloak_nebari_bot_password"]["value"],
             "helm_extensions": [_.model_dump() for _ in self.config.helm_extensions],
+            "forwardauth_middleware_name": stage_outputs[
+                "stages/07-kubernetes-services"
+            ]["forward-auth-middleware"]["value"]["name"],
         }
 
 

src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/locals.tf

@@ -1,6 +1,6 @@
 locals {
   middlewares = (var.private) ? ([{
-    name      = "traefik-forward-auth"
+    name      = var.forwardauth_middleware_name
     namespace = var.namespace
   }]) : ([])
 

src/_nebari/stages/nebari_tf_extensions/template/modules/nebariextension/variables.tf

@@ -70,3 +70,8 @@ variable "keycloak_nebari_bot_password" {
   type        = string
   default     = ""
 }
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}

src/_nebari/stages/nebari_tf_extensions/template/tf-extensions.tf

@@ -16,6 +16,7 @@ module "extension" {
   nebari-realm-id  = var.realm_id
 
   keycloak_nebari_bot_password = each.value.keycloakadmin ? var.keycloak_nebari_bot_password : ""
+  forwardauth_middleware_name  = var.forwardauth_middleware_name
 
   envs = lookup(each.value, "envs", [])
 }

src/_nebari/stages/nebari_tf_extensions/template/variables.tf

@@ -31,3 +31,8 @@ variable "helm_extensions" {
 variable "keycloak_nebari_bot_password" {
   description = "Keycloak password for nebari-bot"
 }
+
+variable "forwardauth_middleware_name" {
+  description = "Name of the traefik forward auth middleware"
+  type        = string
+}