Don't create PVCs if no default StorageClass is set (kubeflow#2679)

* Check for default StorageClass

If no default StorageClass is provided, then we don't let the
users create new Volumes. Neither Workspace nor Data Volumes.

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Use client for creating pvcs

Instead of using hand-made dicts for pvcs, use client
classes for kubernetes objects.

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Fix sharp corners

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Add helper functions

Function for
* Loading the rok secret
* Decoding the parameterized ROK_SECRET_NAME with a user value

The user value will currently always be 'user'

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Handle the token from the html

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Refactor code. Cleanup post_notebook()

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Add permissions to list storage classes

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Add logging

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Reorganize code

* Create baseui module to reduce code duplication
* Edit Makefile to run the webapps

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Update the Dockerfile

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Add prodonjs and avdaredevil as Reviewers

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Add comments

* Add more comments to make the code more descriptive
* Replace single quotes with double ones
* Add readme for baseui module

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

* Sort OWNERS

Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>

Dockerfile

@@ -12,6 +12,9 @@ RUN pip3 install -r /app/requirements.txt
 
 COPY default /app/default
 COPY rok /app/rok
+COPY baseui /app/baseui
+
+ENV PYTHONPATH /app
 
 WORKDIR /app/default
 

components/jupyter-web-app/Makefile

@@ -1,38 +1,10 @@
-IMG = gcr.io/kubeflow-images-public/jupyter-web-app
+# Load the shared modules dir to PYTHONPATH
+MODULE_DIR := $(pwd)
 
-# List any changed  files. We only include files in the notebooks directory.
-# because that is the code in the docker image.
-# In particular we exclude changes to the ksonnet configs.
-CHANGED_FILES := $(shell git diff-files --relative=components/jupyter-web-app)
+all: run-default
 
-ifeq ($(strip $(CHANGED_FILES)),)
-# Changed files is empty; not dirty
-# Don't include --dirty because it could be dirty if files outside the ones we care
-# about changed.
-GIT_VERSION := $(shell git describe --always)
-else
-GIT_VERSION := $(shell git describe --always)-dirty-$(shell git diff | shasum -a256 | cut -c -6)
-endif
+run-default:
+	cd default && PYTHONPATH=${PYTHONPATH}:${MODULE_DIR} python run.py
 
-TAG := $(shell date +v%Y%m%d)-$(GIT_VERSION)
-all: build
-
-# To build without the cache set the environment variable
-# export DOCKER_BUILD_OPTS=--no-cache
-build:
-	docker build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) . \
-	  --build-arg kubeflowversion=$(shell git describe --abbrev=0 --tags) \
-      --label=git-verions=$(GIT_VERSION)
-	docker tag $(IMG):$(TAG) $(IMG):latest
-	@echo Built $(IMG):latest
-	@echo Built $(IMG):$(TAG)
-
-# Build but don't attach the latest tag. This allows manual testing/inspection of the image
-# first.
-push: build
-	gcloud docker -- push $(IMG):$(TAG)
-	@echo Pushed $(IMG) with  :$(TAG) tags
-
-push-latest: push
-	gcloud container images add-tag --quiet $(IMG):$(TAG) $(IMG):latest --verbosity=info
-	echo created $(IMG):latest
+run-rok:
+	cd rok && PYTHONPATH=${PYTHONPATH}:${MODULE_DIR} python run.py

components/jupyter-web-app/OWNERS

@@ -2,4 +2,6 @@ approvers:
   - ioandr
   - kimwnasptd
 reviewers:
+  - avdaredevil
+  - prodonjs
   - vkoukis

components/jupyter-web-app/baseui/.gitignore

@@ -0,0 +1 @@
+/__pycache__

components/jupyter-web-app/baseui/README.md

@@ -0,0 +1,4 @@
+### BaseUI: Shared functions and Utilities
+This module holds functions/configs that both the Default and Rok Jupyter UIs use. Each UI will need to handle its own routes. 
+
+This module is loaded into the `default` and `rok` UIs with the `PYTHONPATH` environment variable. To test the UIs locally, use the corresponding `make` commands.

components/jupyter-web-app/baseui/api.py

@@ -0,0 +1,144 @@
+import json
+from kubernetes import client, config
+from kubernetes.config import ConfigException
+from baseui.utils import create_logger
+
+logger = create_logger(__name__)
+
+try:
+  # Load configuration inside the Pod
+  config.load_incluster_config()
+except ConfigException:
+  # Load configuration for testing
+  config.load_kube_config()
+
+# Create the Apis
+v1_core = client.CoreV1Api()
+custom_api = client.CustomObjectsApi()
+storage_api = client.StorageV1Api()
+
+
+def parse_error(e):
+  try:
+    err = json.loads(e.body)["message"]
+  except json.JSONDecodeError:
+    err = str(e)
+  except KeyError:
+    err = str(e)
+
+  return err
+
+
+def create_workspace_pvc(body):
+  # body: Dict (request body)
+  """If the type is New, then create a new PVC, else use an existing one"""
+  if body["ws_type"] == "New":
+    pvc = client.V1PersistentVolumeClaim(
+        metadata=client.V1ObjectMeta(
+            name=body["ws_name"],
+            namespace=body["ns"]
+        ),
+        spec=client.V1PersistentVolumeClaimSpec(
+            access_modes=[body["ws_access_modes"]],
+            resources=client.V1ResourceRequirements(
+                requests={
+                    "storage": body["ws_size"] + "Gi"
+                }
+            )
+        )
+    )
+
+    create_pvc(pvc)
+
+  return
+
+
+def create_datavol_pvc(body, i):
+  # body: Dict (request body)
+  pvc_nm = body["vol_name" + i]
+
+  # Create a PVC if its a new Data Volume
+  if body["vol_type" + i] == "New":
+    size = body["vol_size" + i] + "Gi"
+    mode = body["vol_access_modes" + i]
+
+    pvc = client.V1PersistentVolumeClaim(
+        metadata=client.V1ObjectMeta(
+            name=pvc_nm,
+            namespace=body["ns"]
+        ),
+        spec=client.V1PersistentVolumeClaimSpec(
+            access_modes=[mode],
+            resources=client.V1ResourceRequirements(
+                requests={
+                    "storage": size
+                }
+            )
+        )
+    )
+
+    create_pvc(pvc)
+
+  return
+
+
+def get_secret(nm, ns):
+  # nm: string
+  # ns: string
+  return v1_core.read_namespaced_secret(nm, ns)
+
+
+def get_default_storageclass():
+  strg_classes = storage_api.list_storage_class().items
+  for strgclss in strg_classes:
+    annotations = strgclss.metadata.annotations
+    # List of possible annotations
+    keys = []
+    keys.append("storageclass.kubernetes.io/is-default-class")
+    keys.append("storageclass.beta.kubernetes.io/is-default-class")  # GKE
+
+    for key in keys:
+      is_default = annotations.get(key, False)
+      if is_default:
+        return strgclss.metadata.name
+
+  # No StorageClass is default
+  return ""
+
+
+def get_namespaces():
+  nmsps = v1_core.list_namespace()
+  return [ns.metadata.name for ns in nmsps.items]
+
+
+def get_notebooks(ns):
+  # ns: string
+  custom_api = client.CustomObjectsApi()
+
+  notebooks = \
+      custom_api.list_namespaced_custom_object("kubeflow.org", "v1alpha1",
+                                               ns, "notebooks")
+  return [nb["metadata"]["name"] for nb in notebooks["items"]]
+
+
+def delete_notebook(nb, ns):
+  # nb: Dict
+  options = client.V1DeleteOptions()
+
+  return \
+      custom_api.delete_namespaced_custom_object("kubeflow.org", "v1alpha1",
+                                                 ns, "notebooks", nb, options)
+
+
+def create_notebook(nb):
+  # nb: Dict
+  ns = nb["metadata"]["namespace"]
+  return \
+      custom_api.create_namespaced_custom_object("kubeflow.org", "v1alpha1",
+                                                 ns, "notebooks", nb)
+
+
+def create_pvc(pvc):
+  # pvc: V1PersistentVolumeClaim
+  ns = pvc.metadata.namespace
+  return v1_core.create_namespaced_persistent_volume_claim(ns, pvc)

components/jupyter-web-app/rok/kubeflow/rokui/utils.py → components/jupyter-web-app/baseui/utils.py

@@ -1,13 +1,24 @@
-# -*- coding: utf-8 -*-
 import yaml
+import logging
+import sys
 
 CONFIG = "/etc/config/spawner_ui_config.yaml"
 
 
+def create_logger(name):
+  handler = logging.StreamHandler(sys.stdout)
+  handler.setFormatter(logging.Formatter(
+      "%(asctime)s | %(name)s | %(levelname)s | %(message)s"))
+  logger = logging.getLogger(name)
+  logger.setLevel(logging.INFO)
+  logger.addHandler(handler)
+  return logger
+
+
 # Functions for handling the JWT token
 def load_file(filepath):
-  with open(filepath, 'r') as f:
-    file_data = f.read().replace('\n', '')
+  with open(filepath, "r") as f:
+    file_data = f.read().replace("\n", "")
 
   return file_data
 
@@ -17,10 +28,10 @@ def load_file(filepath):
 def spawner_ui_config(username):
   c = None
   try:
-    with open(CONFIG, 'r') as f:
+    with open(CONFIG, "r") as f:
       c = f.read().format(username=username)
   except IOError:
-    print('Error opening Spawner UI config file')
+    print("Error opening Spawner UI config file")
 
   try:
     if yaml.safe_load(c) is None:
@@ -33,7 +44,8 @@ def spawner_ui_config(username):
     return None
 
 
-# Helper functions for the /post-notebook route.
+# Since notebook is a CRD, we don't currently have k8s client functions to
+# create the corresponding object.
 def create_notebook_template():
   notebook = {
       "apiVersion": "kubeflow.org/v1alpha1",
@@ -61,59 +73,39 @@ def create_notebook_template():
   return notebook
 
 
-def create_pvc_template():
-  pvc = {
-      "apiVersion": "v1",
-      "kind": "PersistentVolumeClaim",
-      "metadata": {
-          "name": "",
-          "namespace": "",
-      },
-      "spec": {
-          "accessModes": [],
-          "resources": {
-              "requests": {
-                  "storage": ""
-              }
-          },
-      }
-  }
-  return pvc
-
-
 def set_notebook_names(nb, body):
-  nb['metadata']['name'] = body["nm"]
-  nb['metadata']['labels']['app'] = body["nm"]
-  nb['spec']['template']['spec']['containers'][0]['name'] = body["nm"]
-  nb['metadata']['namespace'] = body["ns"]
+  nb["metadata"]["name"] = body["nm"]
+  nb["metadata"]["labels"]["app"] = body["nm"]
+  nb["spec"]["template"]["spec"]["containers"][0]["name"] = body["nm"]
+  nb["metadata"]["namespace"] = body["ns"]
 
 
 def set_notebook_image(nb, body):
   if body["imageType"] == "standard":
     image = body["standardImages"]
   else:
     image = body["customImage"]
-  nb["spec"]['template']['spec']['containers'][0]['image'] = image
+  nb["spec"]["template"]["spec"]["containers"][0]["image"] = image
 
 
 def set_notebook_cpu_ram(nb, body):
-  notebook_cont = nb["spec"]['template']['spec']['containers'][0]
+  notebook_cont = nb["spec"]["template"]["spec"]["containers"][0]
 
-  notebook_cont['resources'] = {
-      'requests': {
-          'cpu': body['cpu'],
-          'memory': body['memory']
+  notebook_cont["resources"] = {
+      "requests": {
+          "cpu": body["cpu"],
+          "memory": body["memory"]
       }
   }
 
 
 def add_notebook_volume(nb, vol, claim, mnt_path):
   # Create the volume in the Pod
-  notebook_spec = nb["spec"]['template']['spec']
-  notebook_cont = nb["spec"]['template']['spec']['containers'][0]
+  notebook_spec = nb["spec"]["template"]["spec"]
+  notebook_cont = nb["spec"]["template"]["spec"]["containers"][0]
 
   volume = {"name": vol, "persistentVolumeClaim": {"claimName": claim}}
-  notebook_spec['volumes'].append(volume)
+  notebook_spec["volumes"].append(volume)
 
   # Container volumeMounts
   mnt = {"mountPath": mnt_path, "name": vol}