Merge pull request #45 from lsst-sqre/tickets/DM-29647

[DM-29647] Helm and chart cleanup

dev-values.yaml

@@ -12,68 +12,9 @@ jupyterhub:
       tag: dev
       pullPolicy: Never
 
-    extraConfig:
-      nublado.py: |
-        import nublado2.hub_config
-        nublado2.hub_config.HubConfig().configure(c)
-
   singleuser:
-    cmd: "/opt/lsst/software/jupyterlab/provisionator.bash"
     storage:
       type: none
-      extraVolumes:
-        - name: tmp
-          emptyDir: {}
-        - name: home
-          emptyDir: {}
-        - name: lab-environment
-          configMap:
-            defaultMode: 420
-            name: lab-environment
-        - name: dask
-          configMap:
-            name: dask
-        - name: passwd
-          configMap:
-            defaultMode: 420
-            name: passwd
-        - name: group
-          configMap:
-            defaultMode: 420
-            name: group
-        - name: shadow
-          configMap:
-            defaultMode: 384
-            name: shadow
-        - name: gshadow
-          configMap:
-            defaultMode: 384
-            name: gshadow
-      extraVolumeMounts:
-        - name: dask
-          mountPath: /etc/dask
-        - name: tmp
-          mountPath: /tmp
-        - name: lab-environment
-          mountPath: /opt/lsst/software/jupyterlab/environment
-        - name: home
-          mountPath: /home
-        - name: passwd
-          mountPath: /etc/passwd
-          readOnly: true
-          subPath: passwd
-        - name: group
-          mountPath: /etc/group
-          readOnly: true
-          subPath: group
-        - name: shadow
-          mountPath: /etc/shadow
-          readOnly: true
-          subPath: shadow
-        - name: gshadow
-          mountPath: /etc/gshadow
-          readOnly: true
-          subPath: gshadow
     initContainers:
       - name: junecleaver
         image: lsstsqre/inittmpdirhome:latest
@@ -92,228 +33,12 @@ jupyterhub:
 config:
   base_url: "https://minikube.lsst.codes"
   images_url: "http://cachemachine.cachemachine.svc.cluster.local/cachemachine/jupyter/available"
-  pinned_images: []
-  sizes:
-    - name: Tiny
-      cpu: 0.5
-      ram: 1536M
-    - name: Small
-      cpu: 1
-      ram: 3072M
-    - name: Medium
-      cpu: 2
-      ram: 6144M
-    - name: Large
-      cpu: 4
-      ram: 12288M
-  user_resources:
-    - apiVersion: v1
-      kind: Namespace
-      metadata:
-        name: "{{ user_namespace }}"
-    - apiVersion: v1
-      kind: ConfigMap
-      metadata:
-        name: lab-environment
-        namespace: "{{ user_namespace }}"
-      data:
-        EXTERNAL_INSTANCE_URL: "{{ base_url }}"
-        FIREFLY_ROUTE: /portal/app
-        HUB_ROUTE: /n2
-        JS9_ROUTE: /js9
-        API_ROUTE: /api
-        TAP_ROUTE: /api/tap
-        SODA_ROUTE: /api/image/soda
-        WORKFLOW_ROUTE: /wf
-        NO_SUDO: 'TRUE'
-        AUTO_REPO_URLS: "https://github.com/lsst-sqre/notebook-demo"
-        EXTERNAL_GROUPS: "{{ external_groups }}"
-        EXTERNAL_USER: "{{ user }}"
-        EXTERNAL_UID: "{{ uid }}"
-        ACCESS_TOKEN: "{{ token }}"
-        IMAGE_DIGEST: "{{ options.image_info.digest }}"
-        IMAGE_DESCRIPTION: "{{ options.image_info.display_name }}"
-        CLEAR_DOTLOCAL: "{{ options.clear_dotlocal }}"
-        DEBUG: "{{ options.debug }}"
-    - apiVersion: v1
-      kind: ConfigMap
-      metadata:
-        name: group
-        namespace: "{{ user_namespace }}"
-      data:
-        group: |
-          root:x:0:
-          bin:x:1:
-          daemon:x:2:
-          sys:x:3:
-          adm:x:4:
-          tty:x:5:
-          disk:x:6:
-          lp:x:7:
-          mem:x:8:
-          kmem:x:9:
-          wheel:x:10:
-          cdrom:x:11:
-          mail:x:12:
-          man:x:15:
-          dialout:x:18:
-          floppy:x:19:
-          games:x:20:
-          tape:x:33:
-          video:x:39:
-          ftp:x:50:
-          lock:x:54:
-          audio:x:63:
-          nobody:x:99:
-          users:x:100:
-          utmp:x:22:
-          utempter:x:35:
-          input:x:999:
-          systemd-journal:x:190:
-          systemd-network:x:192:
-          dbus:x:81:
-          ssh_keys:x:998:
-          lsst_lcl:x:1000:{{ user }}
-          tss:x:59:
-          cgred:x:997:
-          screen:x:84:
-          jovyan:x:768:{{ user }}
-          provisionator:x:769:
-          {{user}}:x:{{uid}}:{% for group in groups %}
-          {{ group.name }}:x:{{ group.id }}:{{ user }}{% endfor %}
-    - apiVersion: v1
-      kind: ConfigMap
-      metadata:
-        name: gshadow
-        namespace: "{{ user_namespace }}"
-      data:
-        gshadow: |
-          root:!::
-          bin:!::
-          daemon:!::
-          sys:!::
-          adm:!::
-          tty:!::
-          disk:!::
-          lp:!::
-          mem:!::
-          kmem:!::
-          wheel:!::
-          cdrom:!::
-          mail:!::
-          man:!::
-          dialout:!::
-          floppy:!::
-          games:!::
-          tape:!::
-          video:!::
-          ftp:!::
-          lock:!::
-          audio:!::
-          nobody:!::
-          users:!::
-          utmp:!::
-          utempter:!::
-          input:!::
-          systemd-journal:!::
-          systemd-network:!::
-          dbus:!::
-          ssh_keys:!::
-          lsst_lcl:!::{{ user }}
-          tss:!::
-          cgred:!::
-          screen:!::
-          jovyan:!::{{ user }}
-          provisionator:!::
-          {{ user }}:!::{% for g in groups %}
-          {{ g.name }}:!::{{ user }}{% endfor %}
-    - apiVersion: v1
-      kind: ConfigMap
-      metadata:
-        name: passwd
-        namespace: "{{ user_namespace }}"
-      data:
-        passwd: |
-          root:x:0:0:root:/root:/bin/bash
-          bin:x:1:1:bin:/bin:/sbin/nologin
-          daemon:x:2:2:daemon:/sbin:/sbin/nologin
-          adm:x:3:4:adm:/var/adm:/sbin/nologin
-          lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
-          sync:x:5:0:sync:/sbin:/bin/sync
-          shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
-          halt:x:7:0:halt:/sbin:/sbin/halt
-          mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
-          operator:x:11:0:operator:/root:/sbin/nologin
-          games:x:12:100:games:/usr/games:/sbin/nologin
-          ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
-          nobody:x:99:99:Nobody:/:/sbin/nologin
-          systemd-network:x:192:192:systemd Network Management:/:/sbin/nologin
-          dbus:x:81:81:System message bus:/:/sbin/nologin
-          lsst_lcl:x:1000:1000::/home/lsst_lcl:/bin/bash
-          tss:x:59:59:Account used by the trousers package to sandbox the tcsd daemon:/dev/null:/sbin/nologin
-          provisionator:x:769:769:Lab provisioning user:/home/provisionator:/bin/bash
-          {{ user }}:x:{{ uid }}:{{ uid }}::/home/{{ user }}:/bin/bash
-    - apiVersion: v1
-      kind: ConfigMap
-      metadata:
-        name: shadow
-        namespace: "{{ user_namespace }}"
-      data:
-        shadow: |
-          root:*:18000:0:99999:7:::
-          bin:*:18000:0:99999:7:::
-          daemon:*:18000:0:99999:7:::
-          adm:*:18000:0:99999:7:::
-          lp:*:18000:0:99999:7:::
-          sync:*:18000:0:99999:7:::
-          shutdown:*:18000:0:99999:7:::
-          halt:*:18000:0:99999:7:::
-          mail:*:18000:0:99999:7:::
-          operator:*:18000:0:99999:7:::
-          games:*:18000:0:99999:7:::
-          ftp:*:18000:0:99999:7:::
-          nobody:*:18000:0:99999:7:::
-          systemd-network:*:18000:0:99999:7:::
-          dbus:*:18000:0:99999:7:::
-          lsst_lcl:*:18000:0:99999:7:::
-          tss:*:18000:0:99999:7:::
-          provisionator:*:18000:0:99999:7:::
-          {{user}}:*:18000:0:99999:7:::
-    - apiVersion: v1
-      kind: ConfigMap
-      metadata:
-        name: dask
-        namespace: "{{ user_namespace }}"
-      data:
-        dask_worker.yml: |
-          {{ dask_yaml | indent(4) }}
-    - apiVersion: v1
-      kind: ServiceAccount
-      metadata:
-        name: "{{ user }}-serviceaccount"
-        namespace: "{{ user_namespace }}"
-    - apiVersion: rbac.authorization.k8s.io/v1
-      kind: Role
-      metadata:
-        name: "{{ user }}-role"
-        namespace: "{{ user_namespace }}"
-      rules:
-      - apiGroups: [""]
-        resources: ["pods"]
-        verbs: ["create", "get", "delete", "list"]
-    - apiVersion: rbac.authorization.k8s.io/v1
-      kind: RoleBinding
-      metadata:
-        name: "{{ user }}-rolebinding"
-        namespace: "{{ user_namespace }}"
-      roleRef:
-        apiGroup: rbac.authorization.k8s.io
-        kind: Role
-        name: "{{ user }}-role"
-      subjects:
-      - kind: ServiceAccount
-        name: "{{ user }}-serviceaccount"
-        namespace: "{{ user_namespace }}"
+  volumes:
+    - name: home
+      emptyDir: {}
+  volume_mounts:
+    - name: home
+      mountPath: /home
 
 vault_secret_path: "secret/k8s_operator/minikube.lsst.codes/nublado2"
 gafaelfawr_secret_path: "secret/k8s_operator/minikube.lsst.codes/gafaelfawr"

src/nublado2/hub_config.py

@@ -57,6 +57,14 @@ def configure(self, c: JupyterHub) -> None:
         # many iterations.
         c.KubeSpawner.image_pull_policy = "Always"
 
+        # Helm is pretty weird about not being able to merge dicts
+        # together.  We use the extraVolumes and extraVolumeMounts
+        # for the standard things we create.  We use this other section
+        # to add more volumes without needing to copy the entire set
+        # of volumes and mounts everywhere we use it.
+        c.KubeSpawner.volumes.extend(nc.volumes)
+        c.KubeSpawner.volume_mounts.extend(nc.volume_mounts)
+
         self.log.info("JupyterHub configuration complete")
         self.log.debug(f"JupyterHub configuration is now: {c}")
 

src/nublado2/nublado_config.py

@@ -2,7 +2,7 @@
 
 __all__ = ["NubladoConfig"]
 
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List
 
 from ruamel import yaml
 from ruamel.yaml import RoundTripLoader
@@ -60,6 +60,14 @@ def sizes(self) -> Dict[str, LabSize]:
         return dict(self._sizes)
 
     @property
-    def user_resources(self) -> Tuple[Any, ...]:
+    def user_resources_template(self) -> str:
         """Retrieve a copy of the lab resources templates."""
-        return tuple(self._config.get("user_resources", []))
+        return self._config.get("user_resources_template")
+
+    @property
+    def volumes(self) -> List[Dict[str, Any]]:
+        return list(self._config["volumes"])
+
+    @property
+    def volume_mounts(self) -> List[Dict[str, Any]]:
+        return list(self._config["volume_mounts"])

src/nublado2/resourcemgr.py

@@ -10,7 +10,7 @@
 from kubernetes import client, config
 from kubernetes.utils import create_from_dict
 from ruamel import yaml
-from ruamel.yaml import RoundTripDumper, RoundTripLoader
+from ruamel.yaml import RoundTripLoader
 from traitlets.config import LoggingConfigurable
 
 from nublado2.nublado_config import NubladoConfig
@@ -65,19 +65,23 @@ async def create_user_resources(
                 "base_url": nc.base_url,
                 "dask_yaml": await self._build_dask_template(spawner),
                 "options": options,
+                "labels": spawner.common_labels,
+                "annotations": spawner.extra_annotations,
             }
 
             self.log.debug(f"Template values={template_values}")
-            for r in nc.user_resources:
-                t_yaml = yaml.dump(r, Dumper=RoundTripDumper)
-                self.log.debug(f"Resource template:\n{t_yaml}")
-                t = Template(t_yaml)
-                templated_yaml = t.render(template_values)
-                self.log.debug(f"Creating resource:\n{templated_yaml}")
-                templated_resource = yaml.load(
-                    templated_yaml, Loader=RoundTripLoader
-                )
-                create_from_dict(self.k8s_api, templated_resource)
+            t = Template(nc.user_resources_template)
+            templated_user_resources = t.render(template_values)
+            self.log.debug("Generated user resources:")
+            self.log.debug(templated_user_resources)
+
+            user_resources = yaml.load(
+                templated_user_resources, Loader=RoundTripLoader
+            )
+
+            for r in user_resources:
+                self.log.debug(f"Creating: {r}")
+                create_from_dict(self.k8s_api, r)
         except Exception:
             self.log.exception("Exception creating user resource!")
             raise