diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..de288e1e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "black" +} \ No newline at end of file diff --git a/jupyterlab_wipp/jupyterlab_wipp/__init__.py b/jupyterlab_wipp/jupyterlab_wipp/__init__.py index b7236829..aeecc1b4 100644 --- a/jupyterlab_wipp/jupyterlab_wipp/__init__.py +++ b/jupyterlab_wipp/jupyterlab_wipp/__init__.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path from ._version import __version__ @@ -8,22 +9,17 @@ with (HERE / "labextension" / "package.json").open() as fid: data = json.load(fid) -def _jupyter_labextension_paths(): - return [{ - "src": "labextension", - "dest": data["name"] - }] +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": data["name"]}] from .handlers import setup_handlers -from .wipp import Wipp +from wipp_client import Wipp def _jupyter_server_extension_points(): - return [{ - "module": "jupyterlab_wipp" - }] + return [{"module": "jupyterlab_wipp"}] def _load_jupyter_server_extension(server_app): @@ -34,10 +30,19 @@ def _load_jupyter_server_extension(server_app): server_app: jupyterlab.labapp.LabApp JupyterLab application instance """ + wipp_ui_url = os.getenv("WIPP_UI_URL") if "WIPP_UI_URL" in os.environ else "" + + server_app.web_app.settings["wipp_urls"] = { + "notebooks_ui_url": os.path.join(wipp_ui_url, "notebooks/"), + "imagescollections_ui_url": os.path.join(wipp_ui_url, "images-collections/"), + "imagescollection_ui_url": os.path.join(wipp_ui_url, "images-collection/"), + "csvcollections_ui_url": os.path.join(wipp_ui_url, "csv-collections/"), + } + server_app.web_app.settings["wipp"] = Wipp() setup_handlers(server_app.web_app) server_app.log.info("Registered jupyterlab_wipp extension at URL path /wipp") + # For backward compatibility with notebook server - useful for Binder/JupyterHub load_jupyter_server_extension = _load_jupyter_server_extension - diff --git a/jupyterlab_wipp/jupyterlab_wipp/handlers.py b/jupyterlab_wipp/jupyterlab_wipp/handlers.py index 6887d0e0..5ddd9237 100644 --- a/jupyterlab_wipp/jupyterlab_wipp/handlers.py +++ b/jupyterlab_wipp/jupyterlab_wipp/handlers.py @@ -1,13 +1,31 @@ +import os +import json +import shutil +import binascii +import requests from jupyter_server.base.handlers import APIHandler from jupyter_server.utils import url_path_join import tornado -import json + + +def gen_random_object_id(): + """ + Generate random ObjectID in MongoDB format + """ + timestamp = "{0:x}".format(int(time.time())) + rest = binascii.b2a_hex(os.urandom(8)).decode("ascii") + return timestamp + rest + class WippHandler(APIHandler): @property def wipp(self): return self.settings["wipp"] + @property + def urls(self): + return self.settings["wipp_urls"] + class InfoCheckHandler(WippHandler): @tornado.web.authenticated @@ -15,19 +33,23 @@ def get(self): response = self.wipp.check_api_is_live() self.finish(json.dumps(response)) + class WippUiUrls(WippHandler): @tornado.web.authenticated def get(self): """ GET request handler, returns relevant WIPP UI URLs """ - self.finish(json.dumps({ - 'root': self.wipp.wipp_ui_url, - 'notebooks': self.wipp.notebooks_ui_url, - 'imagescollections': self.wipp.imagescollections_ui_url, - 'imagescollection': self.wipp.imagescollection_ui_url, - 'csvcollections': self.wipp.csvcollections_ui_url - })) + self.finish( + json.dumps( + { + "notebooks": self.urls["notebooks_ui_url"], + "imagescollections": self.urls["imagescollections_ui_url"], + "imagescollection": self.urls["imagescollection_ui_url"], + "csvcollections": self.urls["csvcollections_ui_url"], + } + ) + ) class WippRegisterNotebook(WippHandler): @@ -45,34 +67,87 @@ def post(self): """ data = json.loads(self.request.body.decode("utf-8")) - if all(key in data for key in ("path","name","description")): + temp_notebooks_path = ( + os.getenv("WIPP_NOTEBOOKS_PATH") + if "WIPP_NOTEBOOKS_PATH" in os.environ + else "/opt/shared/wipp/temp/notebooks" + ) + + if all(key in data for key in ("path", "name", "description")): try: - response = self.wipp.register_notebook(data["path"], data["name"], data["description"]) - self.finish(json.dumps(response)) + api_route = ( + os.getenv("WIPP_API_INTERNAL_URL") + if "WIPP_API_INTERNAL_URL" in os.environ + else "http://wipp-backend:8080/api" + ) + notebooks_api_route = os.path.join(api_route, "notebooks") + + # Append default path + notebook_path = os.path.join(os.environ["HOME"], data["path"]) + + # Generate random ObjectID for notebook + object_id = gen_random_object_id() + + # Create destination folder in WIPP + dest_folder = os.path.join(temp_notebooks_path, object_id) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + # Copy notebook to destination folder in WIPP + dest_path = os.path.join(dest_folder, "notebook.ipynb") + shutil.copy(notebook_path, dest_path) + + # Send API request to WIPP to register notebook + url = os.path.join(notebooks_api_route, "import") + querystring = { + "folderName": object_id, + "name": data["name"], + "description": data["description"], + } + response = requests.request("POST", url, params=querystring) + + result = {"code": response.status_code} + if response.status_code == 200: + response_json = response.json() + + # Append workflow URL information + response_json["url"] = self.notebooks_ui_url + + result["info"] = response_json + elif response.status_code == 400: + result["error"] = response.text + + self.finish(json.dumps(result)) except: self.write_error(500) else: self.write_error(400) + class WippImageCollections(WippHandler): @tornado.web.authenticated def get(self): """ GET request handler, returns an array of WIPP Image Collections """ - + try: - response = self.wipp.get_image_collections() - self.finish(json.dumps(response)) + response = [ + collection.dict(by_alias=True) + for collection in self.wipp.get_image_collections() + ] + + self.finish(json.dumps(response, default=str)) except: self.write_error(500) + class WippImageCollectionsSearch(WippHandler): @tornado.web.authenticated def post(self): """ POST request handler - Returns an array of WIPP Image Collection which have requested string in the name + Returns an array of WIPP Image Collection which have requested string in the name Input format: { @@ -83,13 +158,16 @@ def post(self): data = json.loads(self.request.body.decode("utf-8")) if "name" in data.keys(): try: - response = self.wipp.search_image_collections(data["name"]) - self.finish(json.dumps(response)) + response = [ + collection.dict(by_alias=True) + for collection in self.wipp.search_image_collections(data["name"]) + ] + self.finish(json.dumps(response, default=str)) except: self.write_error(500) else: self.write_error(400) - + class WippCsvCollections(WippHandler): @tornado.web.authenticated @@ -98,17 +176,22 @@ def get(self): GET request handler, returns an array of WIPP Csv Collections """ try: - response = self.wipp.get_csv_collections() - self.finish(json.dumps(response)) + response = [ + collection.dict(by_alias=True) + for collection in self.wipp.get_csv_collections() + ] + + self.finish(json.dumps(response, default=str)) except: self.write(500) + class WippCsvCollectionsSearch(WippHandler): @tornado.web.authenticated def post(self): """ POST request handler - Returns an array of WIPP Csv Collection which have requested string in the name + Returns an array of WIPP Csv Collection which have requested string in the name Input format: { @@ -119,8 +202,11 @@ def post(self): data = json.loads(self.request.body.decode("utf-8")) if "name" in data.keys(): try: - response = self.wipp.search_csv_collections(data["name"]) - self.finish(json.dumps(response)) + response = [ + collection.dict(by_alias=True) + for collection in self.wipp.search_csv_collections(data["name"]) + ] + self.finish(json.dumps(response, default=str)) except: self.write_error(500) else: @@ -129,13 +215,13 @@ def post(self): def setup_handlers(web_app): handlers = [ - ('/wipp/info', InfoCheckHandler), - ('/wipp/ui_urls', WippUiUrls), - ('/wipp/register', WippRegisterNotebook), - ('/wipp/imageCollections', WippImageCollections), - ('/wipp/imageCollections/search', WippImageCollectionsSearch), - ('/wipp/csvCollections', WippCsvCollections), - ('/wipp/csvCollections/search', WippCsvCollectionsSearch) + ("/wipp/info", InfoCheckHandler), + ("/wipp/ui_urls", WippUiUrls), + ("/wipp/register", WippRegisterNotebook), + ("/wipp/imageCollections", WippImageCollections), + ("/wipp/imageCollections/search", WippImageCollectionsSearch), + ("/wipp/csvCollections", WippCsvCollections), + ("/wipp/csvCollections/search", WippCsvCollectionsSearch), ] base_url = web_app.settings["base_url"] diff --git a/jupyterlab_wipp/jupyterlab_wipp/wipp.py b/jupyterlab_wipp/jupyterlab_wipp/wipp.py deleted file mode 100644 index 9948b855..00000000 --- a/jupyterlab_wipp/jupyterlab_wipp/wipp.py +++ /dev/null @@ -1,250 +0,0 @@ -""" -Module for registering Jupyter notebooks in WIPP based on post request from handlers -""" -import os -import shutil -import sys -import time -import binascii -import requests - -import json - -def gen_random_object_id(): - """ - Generate random ObjectID in MongoDB format - """ - timestamp = '{0:x}'.format(int(time.time())) - rest = binascii.b2a_hex(os.urandom(8)).decode('ascii') - return timestamp + rest - -class WippCollection(): - """Class for holding generic WIPP Collection""" - - def __init__(self, json): - self.json = json - - self.id = self.json['id'] - self.name = self.json['name'] - - def __repr__(self): - return f'{self.id}\t{self.name}' - -class Wipp: - """Class for interfacing with WIPP API""" - def __init__(self): - """Wipp class constructor - - Constructor does not take any arguments directly, but rather reads them from environment variables - """ - # WIPP UI URL -- env variable required - self.wipp_ui_url = os.getenv('WIPP_UI_URL') if "WIPP_UI_URL" in os.environ else '' - self.notebooks_ui_url = os.path.join(self.wipp_ui_url, 'notebooks/') - self.imagescollections_ui_url = os.path.join(self.wipp_ui_url, 'images-collections/') - self.imagescollection_ui_url = os.path.join(self.wipp_ui_url, 'images-collection/') - self.csvcollections_ui_url = os.path.join(self.wipp_ui_url, 'csv-collections/') - - # Other configurable variables: if no env variable provided, take default value - self.api_route = os.getenv('WIPP_API_INTERNAL_URL') if "WIPP_API_INTERNAL_URL" in os.environ else 'http://wipp-backend:8080/api' - self.notebooks_path = os.getenv('WIPP_NOTEBOOKS_PATH') if "WIPP_NOTEBOOKS_PATH" in os.environ else "/opt/shared/wipp/temp/notebooks" - - def check_api_is_live(self): - try: - r = requests.get(self.api_route, timeout=1) - except: - return {"code": 500, "data": "WIPP API is not available, so JupyterLab-WIPP extension will not be loaded"} - - if r.status_code==200: - if '_links' in r.json(): - return {"code": 200, "data": "JupyterLab-WIPP extension is loaded"} - - - - def register_notebook(self, notebook_path, name, description): - """Register Notebook in WIPP - - Keyword arguments: - notebook_path -- path to Notebook file relative to HOME (usually returned by JupyterLab context menu) - name -- string with Notebook display name - description -- string with short description of what the Notebook does - """ - - notebooks_api_route = os.path.join(self.api_route, 'notebooks') - notebook_path = os.path.join(os.environ['HOME'], notebook_path) #append default path - - #Generate random ObjectID for notebook - object_id = gen_random_object_id() - - #Create destination folder in WIPP - dest_folder = os.path.join(self.notebooks_path, object_id) - if not os.path.exists(dest_folder): - os.makedirs(dest_folder) - - #Copy notebook to destination folder in WIPP - dest_path = os.path.join(dest_folder, 'notebook.ipynb') - shutil.copy(notebook_path, dest_path) - - #Send API request to WIPP to register notebook - url = os.path.join(notebooks_api_route, 'import') - querystring = { - "folderName":object_id, - "name":name, - "description":description - } - response = requests.request("POST", url, params=querystring) - - result = {"code": response.status_code} - if response.status_code == 200: - response_json = response.json() - - #Append workflow URL information - response_json["url"] = self.notebooks_ui_url - - result["info"] = response_json - elif response.status_code == 400: - result["error"] = response.text - return result - - def get_image_collections_summary(self): - """Get tuple with WIPP's Image Collections number of pages and page size""" - - r = requests.get(os.path.join(self.api_route, 'imagesCollections')) - if r.status_code==200: - total_pages = r.json()['page']['totalPages'] - page_size = r.json()['page']['size'] - - return (total_pages, page_size) - - def get_image_collections_page(self, index): - """Get the page of WIPP Image Collections - - Keyword arguments: - index -- page index starting from 0 - """ - r = requests.get(os.path.join(self.api_route, f'imagesCollections?page={index}')) - if r.status_code==200: - collections_page = r.json()['_embedded']['imagesCollections'] - return [WippCollection(collection) for collection in collections_page] - - def get_image_collections_all_pages(self): - """Get list of all pages of WIPP Image Collections""" - total_pages, _ = self.get_image_collections_summary() - return [self.get_image_collections_page(page) for page in range(total_pages)] - - - def get_image_collections(self): - """Get list of all available WIPP Image Collection in JSON format""" - return [collection.json for collection in sum(self.get_image_collections_all_pages(), [])] - - def search_image_collections_summary(self, name): - """Get tuple with number of pages and page size of WIPP Image Collections that contain search string in the name - - Keyword arguments: - name -- string to search in Image Collection names - """ - r = requests.get(os.path.join(self.api_route, f'imagesCollections/search/findByNameContainingIgnoreCase?name={name}')) - if r.status_code==200: - total_pages = r.json()['page']['totalPages'] - page_size = r.json()['page']['size'] - - return (total_pages, page_size) - - def search_image_collection_page(self, name, index): - """Get the page of WIPP Image Collection search - - Keyword arguments: - name -- string to search in Image Collection names - index -- page index starting from 0 - """ - r = requests.get(os.path.join(self.api_route, f'imagesCollections/search/findByNameContainingIgnoreCase?name={name}&page={index}')) - if r.status_code==200: - collections_page = r.json()['_embedded']['imagesCollections'] - return [WippCollection(collection) for collection in collections_page] - - def search_image_collections_all_pages(self, name): - """Get list of all pages of WIPP Image Collections search - - Keyword arguments: - name -- string to search in Csv Collection names - """ - total_pages, _ = self.search_image_collections_summary(name) - return [self.search_image_collection_page(name, page) for page in range(total_pages)] - - def search_image_collections(self, name): - """Get list of all found WIPP Image Collection in JSON format - - Keyword arguments: - name -- string to search in Csv Collection names - """ - return [collection.json for collection in sum(self.search_image_collections_all_pages(name), [])] - - def get_csv_collections_summary(self): - """Get tuple with WIPP's Csv Collections number of pages and page size""" - r = requests.get(os.path.join(self.api_route, 'csvCollections')) - if r.status_code==200: - total_pages = r.json()['page']['totalPages'] - page_size = r.json()['page']['size'] - - return (total_pages, page_size) - - def get_csv_collections_page(self, index): - """Get the page of WIPP Csv Collections - - Keyword arguments: - index -- page index starting from 0 - """ - r = requests.get(os.path.join(self.api_route, f'csvCollections?page={index}')) - if r.status_code==200: - collections_page = r.json()['_embedded']['csvCollections'] - return [WippCollection(collection) for collection in collections_page] - - def get_csv_collections_all_pages(self): - """Get list of all pages of WIPP Csv Collections""" - total_pages, _ = self.get_csv_collections_summary() - return [self.get_csv_collections_page(page) for page in range(total_pages)] - - def get_csv_collections(self): - """Get list of all available WIPP Csv Collection in JSON format""" - return [collection.json for collection in sum(self.get_csv_collections_all_pages(), [])] - - def search_csv_collections_summary(self, name): - """Get tuple with number of pages and page size of WIPP Csv Collections that contain search string in the name - - Keyword arguments: - name -- string to search in Csv Collection names - """ - r = requests.get(os.path.join(self.api_route, f'csvCollections/search/findByNameContainingIgnoreCase?name={name}')) - if r.status_code==200: - total_pages = r.json()['page']['totalPages'] - page_size = r.json()['page']['size'] - - return (total_pages, page_size) - - def search_csv_collection_page(self, name, index): - """Get the page of WIPP Csv Collection search - - Keyword arguments: - name -- string to search in Csv Collection names - index -- page index starting from 0 - """ - r = requests.get(os.path.join(self.api_route, f'csvCollections/search/findByNameContainingIgnoreCase?name={name}&page={index}')) - if r.status_code==200: - collections_page = r.json()['_embedded']['csvCollections'] - return [WippCollection(collection) for collection in collections_page] - - def search_csv_collections_all_pages(self, name): - """Get list of all pages of WIPP Csv Collections search - - Keyword arguments: - name -- string to search in Csv Collection names - """ - total_pages, _ = self.search_csv_collections_summary(name) - return [self.search_csv_collection_page(name, page) for page in range(total_pages)] - - def search_csv_collections(self, name): - """Get list of all found WIPP Csv Collection in JSON format - - Keyword arguments: - name -- string to search in Csv Collection names - """ - return [collection.json for collection in sum(self.search_csv_collections_all_pages(name), [])] diff --git a/jupyterlab_wipp/package.json b/jupyterlab_wipp/package.json index 5abd753e..2e0db460 100644 --- a/jupyterlab_wipp/package.json +++ b/jupyterlab_wipp/package.json @@ -1,6 +1,6 @@ { "name": "jupyterlab_wipp", - "version": "1.2.0", + "version": "1.3.0", "description": "WIPP integration with JupyterLab", "keywords": [ "jupyter", diff --git a/jupyterlab_wipp/setup.py b/jupyterlab_wipp/setup.py index 68e66b33..7dcfacda 100644 --- a/jupyterlab_wipp/setup.py +++ b/jupyterlab_wipp/setup.py @@ -5,8 +5,10 @@ from pathlib import Path import setuptools + try: from jupyter_packaging import get_data_files, npm_builder, wrap_installers + try: import jupyterlab except ImportError as e: @@ -21,25 +23,27 @@ # The name of the project name = "jupyterlab_wipp" -lab_path = (HERE / name.replace("-", "_") / "labextension") +lab_path = HERE / name.replace("-", "_") / "labextension" # Representative files that should exist after a successful build -ensured_targets = [ - str(lab_path / "package.json"), - str(lab_path / "static/style.js") -] +ensured_targets = [str(lab_path / "package.json"), str(lab_path / "static/style.js")] labext_name = "jupyterlab_wipp" data_files_spec = [ ("share/jupyter/labextensions/%s" % labext_name, str(lab_path), "**"), ("share/jupyter/labextensions/%s" % labext_name, str(HERE), "install.json"), - ("etc/jupyter/jupyter_server_config.d", - "jupyter-config/server-config", "jupyterlab_wipp.json"), + ( + "etc/jupyter/jupyter_server_config.d", + "jupyter-config/server-config", + "jupyterlab_wipp.json", + ), # For backward compatibility with notebook server - ("etc/jupyter/jupyter_notebook_config.d", - "jupyter-config/nb-config", "jupyterlab_wipp.json"), - + ( + "etc/jupyter/jupyter_notebook_config.d", + "jupyter-config/nb-config", + "jupyterlab_wipp.json", + ), ] long_description = (HERE / "README.md").read_text() @@ -63,9 +67,7 @@ long_description_content_type="text/markdown", packages=setuptools.find_packages(), data_files=get_data_files(data_files_spec), - install_requires=[ - "jupyter_server>=1.6,<2" - ], + install_requires=["jupyter_server>=1.6,<2", "wipp-client>0.2.1"], zip_safe=False, include_package_data=True, python_requires=">=3.6", @@ -81,7 +83,9 @@ "Programming Language :: Python :: 3.9", "Framework :: Jupyter", ], - cmdclass=wrap_installers(post_develop=post_develop, ensured_targets=ensured_targets) + cmdclass=wrap_installers( + post_develop=post_develop, ensured_targets=ensured_targets + ), ) if __name__ == "__main__": diff --git a/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/__init__.py b/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/__init__.py index b35acfe8..df15c80e 100644 --- a/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/__init__.py +++ b/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/__init__.py @@ -2,7 +2,7 @@ from pathlib import Path from ._version import __version__ -from wipp_client.wipp import Wipp +from wipp_client import Wipp HERE = Path(__file__).parent.resolve() diff --git a/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/handlers.py b/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/handlers.py index 5e7719d0..0b9f97b5 100644 --- a/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/handlers.py +++ b/jupyterlab_wipp_plugin_creator/jupyterlab_wipp_plugin_creator/handlers.py @@ -1,5 +1,7 @@ import json import os +import time +import binascii from shutil import copy2 from kubernetes import client, config @@ -9,11 +11,17 @@ from jinja2 import Template import tornado -from wipp_client.wipp import gen_random_object_id from .log import get_logger logger = get_logger() +def gen_random_object_id(): + """ + Generate random ObjectID in MongoDB format + """ + timestamp = '{0:x}'.format(int(time.time())) + rest = binascii.b2a_hex(os.urandom(8)).decode('ascii') + return timestamp + rest def setup_k8s_api(): """ @@ -138,7 +146,7 @@ def post(self): ) else: try: - self.wipp.register_plugin(form) + self.wipp.create_plugin(form) logger.info("WIPP plugin registered!") except Exception as e: logger.error("WIPP plugin register failed,", exc_info=e) diff --git a/jupyterlab_wipp_plugin_creator/package.json b/jupyterlab_wipp_plugin_creator/package.json index 4f9e3b0a..34b712d3 100644 --- a/jupyterlab_wipp_plugin_creator/package.json +++ b/jupyterlab_wipp_plugin_creator/package.json @@ -1,6 +1,6 @@ { "name": "jupyterlab_wipp_plugin_creator", - "version": "0.2.4", + "version": "0.2.6", "description": "Create WIPP container for local Python code", "keywords": [ "jupyter", diff --git a/jupyterlab_wipp_plugin_creator/setup.py b/jupyterlab_wipp_plugin_creator/setup.py index 2b0aae3a..11d3e2bf 100644 --- a/jupyterlab_wipp_plugin_creator/setup.py +++ b/jupyterlab_wipp_plugin_creator/setup.py @@ -55,7 +55,7 @@ long_description=long_description, long_description_content_type="text/markdown", packages=setuptools.find_packages(), - install_requires=["jupyter_server>=1.6,<2", "wipp-client", "kubernetes"], + install_requires=["jupyter_server>=1.6,<2", "wipp-client>0.2.1", "kubernetes"], zip_safe=False, include_package_data=True, python_requires=">=3.6",