From 045d11262dc7aab4d542ed195323a5fdbdd92f1a Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Thu, 7 Apr 2022 07:16:53 +0530 Subject: [PATCH 1/3] Suport adding load balancer annotations and ip via terraform overrides --- qhub/schema.py | 10 ++++++++++ qhub/stages/input_vars.py | 4 +++- .../stages/04-kubernetes-ingress/main.tf | 10 ++++++---- .../modules/kubernetes/ingress/main.tf | 5 +++-- .../modules/kubernetes/ingress/variables.tf | 15 +++++++++++++++ .../stages/04-kubernetes-ingress/variables.tf | 17 +++++++++++++++++ 6 files changed, 54 insertions(+), 7 deletions(-) diff --git a/qhub/schema.py b/qhub/schema.py index 10b5918f1..535f62efa 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -382,6 +382,15 @@ class QHubExtension(Base): envs: typing.Optional[typing.List[QHubExtensionEnv]] +class IngressTerraformOverrides(Base): + load_balancer_annotations: typing.Optional[typing.Dict] + load_balancer_ip: str + + +class Ingress(Base): + terraform_overrides: IngressTerraformOverrides + + # ======== External Container Registry ======== # This allows the user to set a private AWS ECR as a replacement for @@ -455,6 +464,7 @@ class Main(Base): prevent_deploy: bool = ( False # Optional, but will be given default value if not present ) + ingress: Ingress # If the qhub_version in the schema is old # we must tell the user to first run qhub upgrade diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index b68319f33..36771fbfb 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -149,7 +149,9 @@ def stage_04_kubernetes_ingress(stage_outputs, config): "enable-certificates": (config["certificate"]["type"] == "lets-encrypt"), "acme-email": config["certificate"].get("acme_email"), "acme-server": config["certificate"].get("acme_server"), - "certificate-secret-name": config["certificate"]["secret_name"] + "certificate-secret-name": config["certificate"]["secret_name"], + "load-balancer-annotations": config["ingress"]["terraform_overrides"]["load_balancer_annotations"], + "load-balancer-ip": config["ingress"]["terraform_overrides"]["load-balancer-ip"] if config["certificate"]["type"] == "existing" else None, } diff --git a/qhub/template/stages/04-kubernetes-ingress/main.tf b/qhub/template/stages/04-kubernetes-ingress/main.tf index 44c8387c5..ec2348f9e 100644 --- a/qhub/template/stages/04-kubernetes-ingress/main.tf +++ b/qhub/template/stages/04-kubernetes-ingress/main.tf @@ -5,8 +5,10 @@ module "kubernetes-ingress" { node-group = var.node_groups.general - enable-certificates = var.enable-certificates - acme-email = var.acme-email - acme-server = var.acme-server - certificate-secret-name = var.certificate-secret-name + enable-certificates = var.enable-certificates + acme-email = var.acme-email + acme-server = var.acme-server + certificate-secret-name = var.certificate-secret-name + load-balancer-annotations = var.load-balancer-annotations + load-balancer-ip = var.load-balancer-ip } diff --git a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf index 99a2ff71e..32a284899 100644 --- a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf +++ b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf @@ -64,9 +64,9 @@ resource "kubernetes_service" "main" { } spec { - selector = { + selector = merge({ "app.kubernetes.io/component" = "traefik-ingress" - } + }, var.load-balancer-annotations) port { name = "http" @@ -111,6 +111,7 @@ resource "kubernetes_service" "main" { } type = "LoadBalancer" + load_balancer_ip = var.load-balancer-ip } } diff --git a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf index 6d5eac16e..d30bb9334 100644 --- a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf +++ b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf @@ -58,3 +58,18 @@ variable "certificate-secret-name" { type = string default = null } + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(object({ + key = string + value = string + })) + default = null +} diff --git a/qhub/template/stages/04-kubernetes-ingress/variables.tf b/qhub/template/stages/04-kubernetes-ingress/variables.tf index c3dc006ba..85929c4fb 100644 --- a/qhub/template/stages/04-kubernetes-ingress/variables.tf +++ b/qhub/template/stages/04-kubernetes-ingress/variables.tf @@ -38,3 +38,20 @@ variable "certificate-secret-name" { description = "Kubernetes secret used for certificate" default = "" } + + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(object({ + key = string + value = string + })) + default = null +} From 700c5b40a978579e4870b4f91eea7b3cdc6842c3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Thu, 7 Apr 2022 07:41:08 +0530 Subject: [PATCH 2/3] add documentation for terraform overrides --- docs/source/installation/configuration.md | 30 +++++++++++++++++++++++ qhub/schema.py | 2 +- qhub/stages/input_vars.py | 11 ++++++--- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/docs/source/installation/configuration.md b/docs/source/installation/configuration.md index 9668fa36a..c1a4a96fe 100644 --- a/docs/source/installation/configuration.md +++ b/docs/source/installation/configuration.md @@ -690,6 +690,36 @@ jupyterhub: users: true ``` +## Terraform Overrides + +The QHub configuration file provides a huge number of configuration options for customizing your +QHub Infrastructure, while these options are sufficient for an average user, but aren't +exhaustive by any means. There are still a plenty of things you might want to achieve which +cannot be configured directly by the above mentioned options, hence we've introduced a +new option called terraform overrides (`terraform_overrides`), which lets you override +the values of terraform variables in specific modules/resource. This is a relatively +advance feature and must be used with utmost care and you should really know, what +you're doing. + +Here we describe the overrides supported via QHub config file: + +### Ingress + +You can configure the IP of the load balancer and add annotations for the same via `ingress`'s +terraform overrides, one such example for GCP is: + + +```yaml +ingress: + terraform_overrides: + load_balancer_annotations: + "networking.gke.io/load-balancer-type": "Internal" + "networking.gke.io/internal-load-balancer-subnet": "pre-existing-subnet" + load_balancer_ip: "1.2.3.4" +``` + +This is quite useful for pinning the IP Address of the load balancer. + # Full configuration example ```yaml diff --git a/qhub/schema.py b/qhub/schema.py index 535f62efa..b6aa8f374 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -464,7 +464,7 @@ class Main(Base): prevent_deploy: bool = ( False # Optional, but will be given default value if not present ) - ingress: Ingress + ingress: typing.Optional[Ingress] # If the qhub_version in the schema is old # we must tell the user to first run qhub upgrade diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index 36771fbfb..a4ff28e60 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -142,6 +142,9 @@ def _calculate_note_groups(config): def stage_04_kubernetes_ingress(stage_outputs, config): + ingress_terraform_overrides = config.get("ingress", {}).get( + "terraform_overrides", {} + ) return { "name": config["project_name"], "environment": config["namespace"], @@ -149,11 +152,13 @@ def stage_04_kubernetes_ingress(stage_outputs, config): "enable-certificates": (config["certificate"]["type"] == "lets-encrypt"), "acme-email": config["certificate"].get("acme_email"), "acme-server": config["certificate"].get("acme_server"), - "certificate-secret-name": config["certificate"]["secret_name"], - "load-balancer-annotations": config["ingress"]["terraform_overrides"]["load_balancer_annotations"], - "load-balancer-ip": config["ingress"]["terraform_overrides"]["load-balancer-ip"] + "certificate-secret-name": config["certificate"]["secret_name"] if config["certificate"]["type"] == "existing" else None, + "load-balancer-annotations": ingress_terraform_overrides.get( + "load_balancer_annotations", {} + ), + "load-balancer-ip": ingress_terraform_overrides.get("load-balancer-ip"), } From 506d254cf3a9d9a728ca8302c37479fe683a343c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 11 Apr 2022 17:20:46 +0530 Subject: [PATCH 3/3] make terraform overrides being able to override any variable --- docs/source/installation/configuration.md | 4 ++-- qhub/schema.py | 7 +------ qhub/stages/input_vars.py | 8 +------- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/docs/source/installation/configuration.md b/docs/source/installation/configuration.md index c1a4a96fe..6ca8d1b86 100644 --- a/docs/source/installation/configuration.md +++ b/docs/source/installation/configuration.md @@ -712,10 +712,10 @@ terraform overrides, one such example for GCP is: ```yaml ingress: terraform_overrides: - load_balancer_annotations: + load-balancer-annotations: "networking.gke.io/load-balancer-type": "Internal" "networking.gke.io/internal-load-balancer-subnet": "pre-existing-subnet" - load_balancer_ip: "1.2.3.4" + load-balancer-ip: "1.2.3.4" ``` This is quite useful for pinning the IP Address of the load balancer. diff --git a/qhub/schema.py b/qhub/schema.py index b6aa8f374..498e5eddf 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -382,13 +382,8 @@ class QHubExtension(Base): envs: typing.Optional[typing.List[QHubExtensionEnv]] -class IngressTerraformOverrides(Base): - load_balancer_annotations: typing.Optional[typing.Dict] - load_balancer_ip: str - - class Ingress(Base): - terraform_overrides: IngressTerraformOverrides + terraform_overrides: typing.Any # ======== External Container Registry ======== diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index a4ff28e60..68e718caa 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -142,9 +142,6 @@ def _calculate_note_groups(config): def stage_04_kubernetes_ingress(stage_outputs, config): - ingress_terraform_overrides = config.get("ingress", {}).get( - "terraform_overrides", {} - ) return { "name": config["project_name"], "environment": config["namespace"], @@ -155,10 +152,7 @@ def stage_04_kubernetes_ingress(stage_outputs, config): "certificate-secret-name": config["certificate"]["secret_name"] if config["certificate"]["type"] == "existing" else None, - "load-balancer-annotations": ingress_terraform_overrides.get( - "load_balancer_annotations", {} - ), - "load-balancer-ip": ingress_terraform_overrides.get("load-balancer-ip"), + **config.get("ingress", {}).get("terraform_overrides", {}), }