From 7c484220648ae317385424d07b9bef1ec147cc0a Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Wed, 31 Mar 2021 18:32:00 -0700 Subject: [PATCH 1/3] [DM-29532] Fix debug and add clear .local flag This is a bit of a refactor, but something that should dovetail into what Adam is doing too. --- dev-values.yaml | 6 ++- src/nublado2/hooks.py | 33 ++++------------ src/nublado2/options.py | 6 ++- src/nublado2/resourcemgr.py | 6 +-- src/nublado2/selectedoptions.py | 67 +++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 src/nublado2/selectedoptions.py diff --git a/dev-values.yaml b/dev-values.yaml index a3facbe..6ff2c58 100644 --- a/dev-values.yaml +++ b/dev-values.yaml @@ -132,8 +132,10 @@ config: EXTERNAL_USER: "{{ user }}" EXTERNAL_UID: "{{ uid }}" ACCESS_TOKEN: "{{ token }}" - IMAGE_DIGEST: "{{ image_info.digest }}" - IMAGE_DESCRIPTION: "{{ image_info.display_name }}" + IMAGE_DIGEST: "{{ options.image_info.digest }}" + IMAGE_DESCRIPTION: "{{ options.image_info.display_name }}" + CLEAR_DOTLOCAL: "{{ options.clear_dotlocal }}" + DEBUG: "{{ options.debug }}" - apiVersion: v1 kind: ConfigMap metadata: diff --git a/src/nublado2/hooks.py b/src/nublado2/hooks.py index 8f055c3..7693bc2 100644 --- a/src/nublado2/hooks.py +++ b/src/nublado2/hooks.py @@ -5,8 +5,9 @@ from nublado2.imageinfo import ImageInfo from nublado2.nublado_config import NubladoConfig -from nublado2.options import DROPDOWN_SENTINEL_VALUE, NubladoOptions +from nublado2.options import NubladoOptions from nublado2.resourcemgr import ResourceManager +from nublado2.selectedoptions import SelectedOptions class NubladoHooks(LoggingConfigurable): @@ -15,32 +16,14 @@ def __init__(self) -> None: self.optionsform = NubladoOptions() async def pre_spawn(self, spawner: Spawner) -> None: - user = spawner.user.name - options = spawner.user_options + options = SelectedOptions(spawner.user_options) self.log.debug( - f"Pre-spawn hook called for {user} with options {options}" + f"Pre-spawn called for {spawner.user.name} with options {options}" ) - # Look up what the user selected on the options form. - # Each parameter comes back as a list, even if only one is - # selected. - size_name = options["size"][0] - img_list_str = options["image_list"][0] - img_dropdown_str = options["image_dropdown"][0] - if img_list_str == DROPDOWN_SENTINEL_VALUE: - image_info = ImageInfo.from_packed_string(img_dropdown_str) - else: - image_info = ImageInfo.from_packed_string(img_list_str) - - # Take size and image info, which are returned as form data, - # look up associated values, and configure the spawner. - # This will help set up the created lab pod. - nc = NubladoConfig() - (cpu, ram) = nc.lookup_size(size_name) - spawner.image = image_info.reference - spawner.debug = options.get("debug_enabled", False) - spawner.mem_limit = ram - spawner.cpu_limit = cpu + spawner.image = options.image_info.reference + spawner.mem_limit = options.ram + spawner.cpu_limit = options.cpu auth_state = await spawner.user.get_auth_state() @@ -55,7 +38,7 @@ async def pre_spawn(self, spawner: Spawner) -> None: # which is useful for dask. spawner.service_account = f"{spawner.user.name}-serviceaccount" - await self.resourcemgr.create_user_resources(spawner, image_info) + await self.resourcemgr.create_user_resources(spawner, options) def post_stop(self, spawner: Spawner) -> None: user = spawner.user.name diff --git a/src/nublado2/options.py b/src/nublado2/options.py index 2b45a2b..d55d18d 100644 --- a/src/nublado2/options.py +++ b/src/nublado2/options.py @@ -61,7 +61,11 @@
Enable debug logs
+ name="enable_debug" value="true"> + Enable debug logs
+ + Clear .local directory (caution!)
diff --git a/src/nublado2/resourcemgr.py b/src/nublado2/resourcemgr.py index bafb118..929b1eb 100644 --- a/src/nublado2/resourcemgr.py +++ b/src/nublado2/resourcemgr.py @@ -13,8 +13,8 @@ from ruamel.yaml import RoundTripDumper, RoundTripLoader from traitlets.config import LoggingConfigurable -from nublado2.imageinfo import ImageInfo from nublado2.nublado_config import NubladoConfig +from nublado2.selectedoptions import SelectedOptions config.load_incluster_config() @@ -31,7 +31,7 @@ class ResourceManager(LoggingConfigurable): http_client = aiohttp.ClientSession() async def create_user_resources( - self, spawner: Spawner, image_info: ImageInfo + self, spawner: Spawner, options: SelectedOptions ) -> None: """Create the user resources for this spawning session.""" try: @@ -65,7 +65,7 @@ async def create_user_resources( "base_url": nc.get("base_url"), "dask_yaml": await self._build_dask_template(spawner), "auto_repo_urls": nc.get("auto_repo_urls"), - "image_info": image_info, + "options": options, } self.log.debug(f"Template values={template_values}") diff --git a/src/nublado2/selectedoptions.py b/src/nublado2/selectedoptions.py new file mode 100644 index 0000000..7f1462f --- /dev/null +++ b/src/nublado2/selectedoptions.py @@ -0,0 +1,67 @@ +from typing import Any, Dict + +from nublado2.imageinfo import ImageInfo +from nublado2.options import DROPDOWN_SENTINEL_VALUE +from nublado2.nublado_config import NubladoConfig + + +class SelectedOptions: + """This class parses the returned options form into fields. + + Use this code to add additional options to the spawner form that + get parsed out of the user return form data. That way we can + have strong typing over them, and one place to parse them out.""" + + def __init__(self, options: Dict[str, Any]) -> None: + """Create a SelectedOptions instance from the formdata.""" + + # Each parameter comes back as a list, even if only one is + # selected. + image_list = options["image_list"][0] + image_dropdown = options["image_dropdown"][0] + size_name = options["size"][0] + + if image_list == DROPDOWN_SENTINEL_VALUE: + self._image_info = ImageInfo.from_packed_string(image_dropdown) + else: + self._image_info = ImageInfo.from_packed_string(image_list) + + nc = NubladoConfig() + (self._cpu, self._ram) = nc.lookup_size(size_name) + + self._debug = "TRUE" if "enable_debug" in options else "" + self._clear_dotlocal = "TRUE" if "clear_dotlocal" in options else "" + + @property + def debug(self) -> str: + """String to pass in for the DEBUG environment variable in the lab + + This sets up the nublado lab containers to emit more debug info, + but doesn't work the same way the kubespawner.debug attribute does.""" + return self._debug + + @property + def clear_dotlocal(self) -> str: + """String to pass in for CLEAR_DOTLOCAL variable in the lab. + + This gets rid of the user's .local directory which may + cause issues during startup.""" + return self._clear_dotlocal + + @property + def image_info(self) -> ImageInfo: + """Information on the Docker image to run for the lab.""" + return self._image_info + + @property + def cpu(self) -> float: + """Number of vCPUs for the lab pod. Comes from the size.""" + return self._cpu + + @property + def ram(self) -> str: + """Amount of RAM for the lab pod. + + This is in kubernetes format, like 2g or 2048M, and comes + from the size.""" + return self._ram From 97ee427a0ad4217c18e96e214766b3634d92a71c Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Mon, 5 Apr 2021 17:32:34 -0700 Subject: [PATCH 2/3] [DM-29532] Increase some timeouts --- src/nublado2/hub_config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/nublado2/hub_config.py b/src/nublado2/hub_config.py index 2f7df98..4a4708d 100644 --- a/src/nublado2/hub_config.py +++ b/src/nublado2/hub_config.py @@ -30,6 +30,17 @@ def configure(self, c: JupyterHub) -> None: c.KubeSpawner.enable_user_namespaces = True + # This is how long the hub will wait for a lab pod to start. + # For large images, this also includes the time it takes to + # pull the docker image and start it. + c.KubeSpawner.start_timeout = 10 * 60 # 10 minutes + + # This is how long to wait after the lab pod starts before + # the hub will give up waiting for the lab to start. When + # using the debug flag, sometimes this can take longer than + # the default, which is 30 seconds. + c.KubeSpawner.http_timeout = 90 + # This is put in the lab pod, and tells kubernetes to # use all the key: values found in the lab-environment # configmap as environment variables for the lab From 19ae8ad595b7d39f32ebaf1c99a4d2d7b149b1dd Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Mon, 5 Apr 2021 17:37:12 -0700 Subject: [PATCH 3/3] [DM-29532] Fix imports Reorder some that got out of order in the rebase, as well as unused imports too. --- src/nublado2/hooks.py | 2 -- src/nublado2/selectedoptions.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nublado2/hooks.py b/src/nublado2/hooks.py index 7693bc2..e570bad 100644 --- a/src/nublado2/hooks.py +++ b/src/nublado2/hooks.py @@ -3,8 +3,6 @@ from jupyterhub.spawner import Spawner from traitlets.config import LoggingConfigurable -from nublado2.imageinfo import ImageInfo -from nublado2.nublado_config import NubladoConfig from nublado2.options import NubladoOptions from nublado2.resourcemgr import ResourceManager from nublado2.selectedoptions import SelectedOptions diff --git a/src/nublado2/selectedoptions.py b/src/nublado2/selectedoptions.py index 7f1462f..4f020a7 100644 --- a/src/nublado2/selectedoptions.py +++ b/src/nublado2/selectedoptions.py @@ -1,8 +1,8 @@ from typing import Any, Dict from nublado2.imageinfo import ImageInfo -from nublado2.options import DROPDOWN_SENTINEL_VALUE from nublado2.nublado_config import NubladoConfig +from nublado2.options import DROPDOWN_SENTINEL_VALUE class SelectedOptions: