From f3a1030b888de3bb4b5e0abd0855668c41af4d49 Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Thu, 19 Nov 2020 21:54:08 -0700 Subject: [PATCH 1/3] [DM-27441] Load images list from URL Allow the retrieval of a list of images from the cachemachine service, which also tell it which is the recommended tag. --- dev-values.yaml | 7 +------ src/nublado2/hooks.py | 6 +++--- src/nublado2/nublado_config.py | 9 --------- src/nublado2/options.py | 28 +++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/dev-values.yaml b/dev-values.yaml index 212ebe6..3ae56ee 100644 --- a/dev-values.yaml +++ b/dev-values.yaml @@ -34,13 +34,8 @@ jupyterhub: config: options_form: + images_url: "http://cachemachine-dev.cachemachine-dev.svc.cluster.local/cachemachine/jupyter/available" images: - - name: Recommended - image: registry.hub.docker.com/jupyterhub/k8s-singleuser-sample:0.9.0-n358.h11c2aeb8 - - name: Weekly 44 - image: registry.hub.docker.com/lsstsqre/sciplat-lab:w_2020_44 - - name: Weekly 43 - image: registry.hub.docker.com/lsstsqre/sciplat-lab:w_2020_43 - name: Weekly 42 image: registry.hub.docker.com/lsstsqre/sciplat-lab:w_2020_42 sizes: diff --git a/src/nublado2/hooks.py b/src/nublado2/hooks.py index ed77af5..43ed85e 100644 --- a/src/nublado2/hooks.py +++ b/src/nublado2/hooks.py @@ -29,7 +29,7 @@ def pre_spawn(self, spawner: Spawner) -> None: # This will help set up the created lab pod. nc = NubladoConfig() (cpu, ram) = nc.lookup_size(size_name) - spawner.image = nc.lookup_image_url(image_name) + spawner.image = image_name spawner.debug = options.get("debug_enabled", False) spawner.mem_limit = ram spawner.cpu_limit = cpu @@ -41,7 +41,7 @@ def post_stop(self, spawner: Spawner) -> None: self.log.debug(f"Post stop-hook called for {user}") self.resourcemgr.delete_user_resources(spawner.namespace) - def show_options(self, spawner: Spawner) -> str: + async def show_options(self, spawner: Spawner) -> str: user = spawner.user.name self.log.debug(f"Show options hook called for {user}") - return self.optionsform.show_options_form(spawner) + return await self.optionsform.show_options_form(spawner) diff --git a/src/nublado2/nublado_config.py b/src/nublado2/nublado_config.py index 6ebb35d..4afeaad 100644 --- a/src/nublado2/nublado_config.py +++ b/src/nublado2/nublado_config.py @@ -24,12 +24,3 @@ def lookup_size(self, name: str) -> Tuple[float, str]: return (float(s["cpu"]), s["ram"]) raise ValueError(f"Size {name} not found") - - def lookup_image_url(self, name: str) -> str: - images = self.get()["options_form"]["images"] - - for i in images: - if i["name"] == name: - return i["image"] - - raise ValueError(f"Image url for {name} not found") diff --git a/src/nublado2/options.py b/src/nublado2/options.py index 98af3d1..90e5cd9 100644 --- a/src/nublado2/options.py +++ b/src/nublado2/options.py @@ -1,3 +1,6 @@ +from typing import Dict, List + +from aiohttp import ClientSession from jinja2 import Template from jupyterhub.spawner import Spawner from traitlets.config import LoggingConfigurable @@ -25,7 +28,7 @@ {% for i in images %} {{ i.name }}
@@ -51,10 +54,29 @@ """ ) +# Don't have this be a member of NubladoOptions, we should +# share this connection pool. Also the LoggingConfigurable +# will try to pickle it to json, and it can't pickle a session. +session = ClientSession() + class NubladoOptions(LoggingConfigurable): - def show_options_form(self, spawner: Spawner) -> str: + async def show_options_form(self, spawner: Spawner) -> str: options_config = NubladoConfig().get()["options_form"] - images = options_config["images"] sizes = options_config["sizes"] + + images_url = options_config.get("images_url", "") + images = await self._get_images_from_url(images_url) + images.extend(options_config["images"]) + return options_template.render(images=images, sizes=sizes) + + async def _get_images_from_url(self, url: str) -> List[Dict[str, str]]: + if not url: + return [] + + async with session.get(url) as r: + if r.status != 200: + raise Exception(f"Error {r.status} from {url}") + + return await r.json() From 2e186283010dd704634b590405cb075d46876f21 Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Wed, 2 Dec 2020 13:35:16 -0700 Subject: [PATCH 2/3] [DM-27441] PR feedback Fix up some of my typing with optional values and the image_url for the dev-values.yaml - this changed to correspond with the output from the cachemachine availability endpoint. --- dev-values.yaml | 2 +- src/nublado2/options.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dev-values.yaml b/dev-values.yaml index 3ae56ee..b8cda02 100644 --- a/dev-values.yaml +++ b/dev-values.yaml @@ -37,7 +37,7 @@ config: images_url: "http://cachemachine-dev.cachemachine-dev.svc.cluster.local/cachemachine/jupyter/available" images: - name: Weekly 42 - image: registry.hub.docker.com/lsstsqre/sciplat-lab:w_2020_42 + image_url: registry.hub.docker.com/lsstsqre/sciplat-lab:w_2020_42 sizes: - name: Tiny cpu: .5 diff --git a/src/nublado2/options.py b/src/nublado2/options.py index 90e5cd9..42bc3d3 100644 --- a/src/nublado2/options.py +++ b/src/nublado2/options.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict, List, Optional from aiohttp import ClientSession from jinja2 import Template @@ -65,18 +65,20 @@ async def show_options_form(self, spawner: Spawner) -> str: options_config = NubladoConfig().get()["options_form"] sizes = options_config["sizes"] - images_url = options_config.get("images_url", "") + images_url = options_config.get("images_url") images = await self._get_images_from_url(images_url) images.extend(options_config["images"]) return options_template.render(images=images, sizes=sizes) - async def _get_images_from_url(self, url: str) -> List[Dict[str, str]]: + async def _get_images_from_url( + self, url: Optional[str] + ) -> List[Dict[str, str]]: if not url: return [] - async with session.get(url) as r: - if r.status != 200: - raise Exception(f"Error {r.status} from {url}") + r = await session.get(url) + if r.status != 200: + raise Exception(f"Error {r.status} from {url}") - return await r.json() + return await r.json() From 5d6bfc235bd0f5be8a0d16b8d935441c563dd3c1 Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Fri, 4 Dec 2020 15:54:42 -0700 Subject: [PATCH 3/3] [DM-27441] Use vault-secrets in helm chart Get rid of some more parts of the dev-values.yaml, and fold it into the vault secrets and the values.yaml of the chart. --- dev-values.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/dev-values.yaml b/dev-values.yaml index b8cda02..8882944 100644 --- a/dev-values.yaml +++ b/dev-values.yaml @@ -17,15 +17,6 @@ jupyterhub: storage: type: none - auth: - state: - cryptoKey: "080c6d0eb8d9f84ef5f0860bea71dba6ab2e1be712c7bc8c2eaf92321c9fd968" - - proxy: - secretToken: "3854245f5b48412ee9b7a2c985da6acdc146f80cc4f10eb175f0efa24530399b" - service: - type: ClusterIP - ingress: hosts: ["minikube.lsst.codes"] annotations: @@ -52,3 +43,5 @@ config: cpu: 4 ram: 12288M user_resources: {} + +vault_secret_path: "secret/k8s_operator/minikube.lsst.codes/nublado2"