From 05ac90b759fd8c775f6f0f5fe1e61d0e17ff2e20 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 18 Nov 2021 15:12:02 +0000 Subject: [PATCH 1/8] First pass at config for carbonplan hub on Azure --- config/hubs/carbonplan-azure.cluster.yaml | 186 ++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 config/hubs/carbonplan-azure.cluster.yaml diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml new file mode 100644 index 0000000000..bbf54a80ea --- /dev/null +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -0,0 +1,186 @@ +name: carbonplan-azure +provider: kubeconfig +kubeconfig: + file: secrets/carbonplan-azure.yaml +support: + config: + prometheus: + server: + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 4 + memory: 8Gi + grafana: + ingress: + hosts: + - grafana.carbonplan-azure.2i2c.cloud + tls: + - secretName: grafana-tls + hosts: + - grafana.carbonplan-azure.2i2c.cloud +hubs: + - name: staging + domain: staging.carbonplan-azure.2i2c.cloud + template: daskhub + auth0: + connection: github + config: &carbonPlanHubConfig + basehub: + azureFile: + enabled: true + nfs: + enabled: false + shareCreator: + enabled: false + jupyterhub: + custom: + homepage: + templateVars: + org: + name: Carbon Plan + logo_url: https://pbs.twimg.com/profile_images/1262387945971101697/5q_X3Ruk_400x400.jpg + url: https://carbonplan.org + designed_by: + name: 2i2c + url: https://2i2c.org + operated_by: + name: 2i2c + url: https://2i2c.org + funded_by: + name: Carbon Plan + url: https://carbonplan.org + singleuser: + image: + name: pangeo/pangeo-notebook + tag: latest + profileList: + # The mem-guarantees are here so k8s doesn't schedule other pods + # on these nodes. + - display_name: "Small: E2s v4" + description: "~2 CPU, ~15G RAM" + kubespawner_override: + # Explicitly unset mem_limit, so it overrides the default memory limit we set in + # basehub/values.yaml + mem_limit: null + mem_guarantee: 12G + node_selector: + hub.jupyter.org/node-size: Standard_E2s_v4 + - display_name: "Medium: E4s v4" + description: "~4 CPU, ~30G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 29G + node_selector: + hub.jupyter.org/node-size: Standard_E4s_v4 + - display_name: "Large: E8s v4" + description: "~8 CPU, ~60G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 60G + node_selector: + hub.jupyter.org/node-size: Standard_E8s_v4 + - display_name: "Huge: E32s v4" + description: "~32 CPU, ~256G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 240G + node_selector: + hub.jupyter.org/node-size: Standard_E32s_v4 + - display_name: "Very Huge: M64s v2" + description: "~64 CPU, ~1024G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 990G + node_selector: + hub.jupyter.org/node-size: Standard_M64s_v2 + - display_name: "Very Very Huge: M128s v2" + description: "~128 CPU, ~2048G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 2000G + node_selector: + hub.jupyter.org/node-size: Standard_M182s_v2 + scheduling: + userPlaceholder: + enabled: false + replicas: 0 + userScheduler: + enabled: false + proxy: + chp: + resources: + requests: + cpu: 0.5 + memory: 256Mi + limits: + cpu: 1 + memory: 4Gi + nodeSelector: {} + hub: + resources: + requests: + cpu: 0.5 + memory: 256Mi + limits: + cpu: 1 + memory: 4Gi + allowNamedServers: true + readinessProbe: + enabled: false + nodeSelector: {} + config: + Authenticator: + allowed_users: &users + - + - jhamman + admin_users: *users + dask-gateway: + traefik: + resources: + requests: + cpu: 0.5 + memory: 512Mi + limits: + cpu: 2 + memory: 4Gi + controller: + resources: + requests: + cpu: 0.5 + memory: 512Mi + limits: + cpu: 2 + memory: 4Gi + gateway: + resources: + requests: + cpu: 0.5 + memory: 512Mi + limits: + cpu: 2 + memory: 4Gi + # TODO: figure out a replacement for userLimits. + extraConfig: + idle: | + # timeout after 30 minutes of inactivity + c.KubeClusterConfig.idle_timeout = 1800 + 10-dynamic-subpath: | + import os + pod_namespace = os.environ['POD_NAMESPACE'] + # FIXME: This isn't setting up _shared dirs properly + c.KubeSpawner.volume_mounts = [ + { + "mountPath": "/home/jovyan", + "name": "home", + "subPath": f"{pod_namespace}/{{username}}" + }, + ] + - name: prod + domain: carbonplan-azure.2i2c.cloud + template: daskhub + auth0: + connection: github + config: *carbonPlanHubConfig From 6468e5a8f0035adb910c3db232d897cfcaff870b Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 18 Nov 2021 15:17:52 +0000 Subject: [PATCH 2/8] Add second community rep as hub admin --- config/hubs/carbonplan-azure.cluster.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml index bbf54a80ea..0b6d406d1c 100644 --- a/config/hubs/carbonplan-azure.cluster.yaml +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -136,6 +136,7 @@ hubs: allowed_users: &users - - jhamman + - norlandrhagen admin_users: *users dask-gateway: traefik: From 99d8dbcf30b29f2a9a0db2a7a236bef2e64f46f7 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 19 Nov 2021 10:56:29 +0000 Subject: [PATCH 3/8] Revert "Remove support for authentication by static kubeconfig file" This reverts commit 37597637a6e42173ba1f472ff436fe2b2381737d. --- deployer/hub.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/deployer/hub.py b/deployer/hub.py index edf3948837..098f0a44ca 100644 --- a/deployer/hub.py +++ b/deployer/hub.py @@ -42,6 +42,8 @@ def auth(self): yield from self.auth_gcp() elif self.spec['provider'] == 'aws': yield from self.auth_aws() + elif self.spec['provider'] == 'kubeconfig': + yield from self.auth_kubeconfig() else: raise ValueError(f'Provider {self.spec["provider"]} not supported') @@ -123,6 +125,23 @@ def deploy_support(self): ]) print("Done!") + def auth_kubeconfig(self): + """ + Context manager for authenticating with just a kubeconfig file + + For the duration of the contextmanager, we: + 1. Decrypt the file specified in kubeconfig.file with sops + 2. Set `KUBECONFIG` env var to our decrypted file path, so applications + we call (primarily helm) will use that as config + """ + config = self.spec['kubeconfig'] + config_path = config['file'] + + with decrypt_file(config_path) as decrypted_key_path: + # FIXME: Unset this after our yield + os.environ['KUBECONFIG'] = decrypted_key_path + yield + def auth_aws(self): """ Reads `aws` nested config and temporarily sets environment variables From 313315460eafe22ecac6c83d4a6aef9369c89034 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 19 Nov 2021 11:29:07 +0000 Subject: [PATCH 4/8] Add storage definition to singleuser pod --- config/hubs/carbonplan-azure.cluster.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml index 0b6d406d1c..4ce0ace1f2 100644 --- a/config/hubs/carbonplan-azure.cluster.yaml +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -103,6 +103,12 @@ hubs: mem_guarantee: 2000G node_selector: hub.jupyter.org/node-size: Standard_M182s_v2 + storage: + type: none + extraVolumes: + - name: home + persistentVolumeClaim: + claimName: home-azurefile scheduling: userPlaceholder: enabled: false From 17b490c0c2a20b87f81434f5bfb37dcfb7cde12f Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 19 Nov 2021 14:04:17 +0000 Subject: [PATCH 5/8] Move extraConfig for dynamic subpaths to correct part of hub config --- config/hubs/carbonplan-azure.cluster.yaml | 27 ++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml index 4ce0ace1f2..16e51c6bc8 100644 --- a/config/hubs/carbonplan-azure.cluster.yaml +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -144,6 +144,18 @@ hubs: - jhamman - norlandrhagen admin_users: *users + extraConfig: + 10-dynamic-subpath: | + import os + pod_namespace = os.environ['POD_NAMESPACE'] + # FIXME: This isn't setting up _shared dirs properly + c.KubeSpawner.volume_mounts = [ + { + "mountPath": "/home/jovyan", + "name": "home", + "subPath": f"{pod_namespace}/{{username}}" + }, + ] dask-gateway: traefik: resources: @@ -170,21 +182,6 @@ hubs: cpu: 2 memory: 4Gi # TODO: figure out a replacement for userLimits. - extraConfig: - idle: | - # timeout after 30 minutes of inactivity - c.KubeClusterConfig.idle_timeout = 1800 - 10-dynamic-subpath: | - import os - pod_namespace = os.environ['POD_NAMESPACE'] - # FIXME: This isn't setting up _shared dirs properly - c.KubeSpawner.volume_mounts = [ - { - "mountPath": "/home/jovyan", - "name": "home", - "subPath": f"{pod_namespace}/{{username}}" - }, - ] - name: prod domain: carbonplan-azure.2i2c.cloud template: daskhub From 8d33c519ce8b540e7f0600247485cefdbdf6ddc0 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 19 Nov 2021 16:12:36 +0000 Subject: [PATCH 6/8] Update domain names --- config/hubs/carbonplan-azure.cluster.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml index 16e51c6bc8..f2b61d8e41 100644 --- a/config/hubs/carbonplan-azure.cluster.yaml +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -16,14 +16,14 @@ support: grafana: ingress: hosts: - - grafana.carbonplan-azure.2i2c.cloud + - grafana.azure.carbonplan.2i2c.cloud tls: - secretName: grafana-tls hosts: - - grafana.carbonplan-azure.2i2c.cloud + - grafana.azure.carbonplan.2i2c.cloud hubs: - name: staging - domain: staging.carbonplan-azure.2i2c.cloud + domain: staging.azure.carbonplan.2i2c.cloud template: daskhub auth0: connection: github From 2db6f66cd93aae8d8d17c697ed18a5f39bc5e9b2 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 19 Nov 2021 16:28:33 +0000 Subject: [PATCH 7/8] Update prod's domain --- config/hubs/carbonplan-azure.cluster.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml index f2b61d8e41..9304c71a04 100644 --- a/config/hubs/carbonplan-azure.cluster.yaml +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -183,7 +183,7 @@ hubs: memory: 4Gi # TODO: figure out a replacement for userLimits. - name: prod - domain: carbonplan-azure.2i2c.cloud + domain: azure.carbonplan.2i2c.cloud template: daskhub auth0: connection: github From b7dc696072cfe86b473c493d2357d704aec536f6 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 19 Nov 2021 16:57:27 +0000 Subject: [PATCH 8/8] Update prod hub's domain again... --- config/hubs/carbonplan-azure.cluster.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/hubs/carbonplan-azure.cluster.yaml b/config/hubs/carbonplan-azure.cluster.yaml index 9304c71a04..b57e0389f9 100644 --- a/config/hubs/carbonplan-azure.cluster.yaml +++ b/config/hubs/carbonplan-azure.cluster.yaml @@ -183,7 +183,7 @@ hubs: memory: 4Gi # TODO: figure out a replacement for userLimits. - name: prod - domain: azure.carbonplan.2i2c.cloud + domain: prod.azure.carbonplan.2i2c.cloud template: daskhub auth0: connection: github