From d5223a9299c044c091871ba7c7995525014f7c36 Mon Sep 17 00:00:00 2001 From: Timur Bazhirov Date: Thu, 12 Dec 2024 23:14:30 -0800 Subject: [PATCH] feature: add jupyterlite files todo: add rests --- pyproject.toml | 3 + src/py/mat3ra/utils/jupyterlite/__init__.py | 0 src/py/mat3ra/utils/jupyterlite/enums.py | 12 ++ src/py/mat3ra/utils/jupyterlite/logger.py | 34 +++++ src/py/mat3ra/utils/jupyterlite/packages.py | 160 ++++++++++++++++++++ src/py/mat3ra/utils/jupyterlite/settings.py | 1 + 6 files changed, 210 insertions(+) create mode 100644 src/py/mat3ra/utils/jupyterlite/__init__.py create mode 100644 src/py/mat3ra/utils/jupyterlite/enums.py create mode 100644 src/py/mat3ra/utils/jupyterlite/logger.py create mode 100644 src/py/mat3ra/utils/jupyterlite/packages.py create mode 100644 src/py/mat3ra/utils/jupyterlite/settings.py diff --git a/pyproject.toml b/pyproject.toml index 661e57f..925ae7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,9 @@ extra = [ "tabulate", "jinja2", ] +jupyterlite = [ + "pyyaml", +] dev = [ "pre-commit", "black", diff --git a/src/py/mat3ra/utils/jupyterlite/__init__.py b/src/py/mat3ra/utils/jupyterlite/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/py/mat3ra/utils/jupyterlite/enums.py b/src/py/mat3ra/utils/jupyterlite/enums.py new file mode 100644 index 0000000..a184910 --- /dev/null +++ b/src/py/mat3ra/utils/jupyterlite/enums.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class EnvironmentsEnum(Enum): + PYODIDE = "pyodide" + PYTHON = "python" + + +class SeverityLevelEnum(Enum): + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" diff --git a/src/py/mat3ra/utils/jupyterlite/logger.py b/src/py/mat3ra/utils/jupyterlite/logger.py new file mode 100644 index 0000000..ba98ef3 --- /dev/null +++ b/src/py/mat3ra/utils/jupyterlite/logger.py @@ -0,0 +1,34 @@ +import os +import inspect +from typing import Optional + +from .enums import SeverityLevelEnum + + +def log(message: str, level: Optional[SeverityLevelEnum] = None, force_verbose: Optional[bool] = None): + """ + Log a message based on the VERBOSE flag in the caller's globals(). + + Args: + message (str): The message to log. + level (SeverityLevelEnum, optional): The severity level of the message (e.g., INFO, WARNING, ERROR). + force_verbose (bool, optional): If True, log the message regardless of the VERBOSE flag in globals(). + """ + if force_verbose is True: + should_log = True + elif force_verbose is False: + should_log = False + else: + # Inspect the caller's globals to get VERBOSE flag + frame = inspect.currentframe() + try: + caller_frame = frame.f_back # type: ignore + caller_globals = caller_frame.f_globals # type: ignore + should_log = caller_globals.get("VERBOSE", os.environ.get("VERBOSE", True)) + finally: + del frame # Avoid reference cycles + if should_log: + if level is None: + print(message) + else: + print(f"{level.value}: {message}") diff --git a/src/py/mat3ra/utils/jupyterlite/packages.py b/src/py/mat3ra/utils/jupyterlite/packages.py new file mode 100644 index 0000000..2c35815 --- /dev/null +++ b/src/py/mat3ra/utils/jupyterlite/packages.py @@ -0,0 +1,160 @@ +import json +import os +import re +import sys +from typing import List +from .enums import EnvironmentsEnum +from .logger import log + +# default value for env.HOME from https://pyodide.org/en/stable/usage/api/js-api.html +ENVIRONMENT = EnvironmentsEnum.PYODIDE if os.environ.get("HOME") == "/home/pyodide" else EnvironmentsEnum.PYTHON + +if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + import micropip +elif ENVIRONMENT == EnvironmentsEnum.PYTHON: + import subprocess + + +async def install_init(): + if sys.platform == "emscripten": + import micropip + for package in ["pyyaml"]: + await micropip.install(package) + + +async def install_package_pyodide(pkg: str, verbose: bool = True): + """ + Install a package in a Pyodide environment. + + Args: + pkg (str): The name of the package to install. + verbose (bool): Whether to print the name of the installed package. + """ + is_url = pkg.startswith("http://") or pkg.startswith("https://") or pkg.startswith("emfs:/") + are_dependencies_installed = not is_url + await micropip.install(pkg, deps=are_dependencies_installed) + pkg_name = pkg.split("/")[-1].split("-")[0] if is_url else pkg.split("==")[0] + if verbose: + log(f"Installed {pkg_name}", force_verbose=verbose) + + +def install_package_python(pkg: str, verbose: bool = True): + """ + Install a package in a standard Python environment. + + Args: + pkg (str): The name of the package to install. + verbose (bool): Whether to print the name of the installed package. + """ + subprocess.check_call([sys.executable, "-m", "pip", "install", pkg]) + if verbose: + log(f"Installed {pkg}", force_verbose=verbose) + + +async def install_package(pkg: str, verbose: bool = True): + """ + Install a package in the current environment. + + Args: + pkg (str): The name of the package to install. + verbose (bool): Whether to print the name of the installed package. + """ + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + await install_package_pyodide(pkg, verbose) + elif ENVIRONMENT == EnvironmentsEnum.PYTHON: + install_package_python(pkg, verbose) + + +def get_config_yml_file_path(config_file_path) -> str: + """ + Get the path to the requirements file. + + Returns: + str: The path to the requirements file. + """ + base_path = os.getcwd() + config_file_full_path = os.path.normpath(os.path.join("/drive/", "./config.yml")) + if config_file_path != "": + config_file_full_path = os.path.normpath(os.path.join(base_path, config_file_path)) + return config_file_full_path + + +def get_packages_list(requirements_dict: dict, notebook_name_pattern: str = "") -> List[str]: + """ + Get the list of packages to install based on the requirements file. + + Args: + requirements_dict (dict): The dictionary containing the requirements. + notebook_name_pattern (str): The pattern of the notebook name. + + Returns: + List[str]: The list of packages to install. + """ + packages_default_common = requirements_dict.get("default", {}).get("packages_common", []) + packages_default_environment_specific = ( + requirements_dict.get("default", {}).get(f"packages_{ENVIRONMENT.value}", []) + ) + + matching_notebook_requirements_list = [cfg for cfg in requirements_dict.get("notebooks", []) if + re.search(cfg.get("name"), notebook_name_pattern)] + packages_notebook_common = [] + packages_notebook_environment_specific = [] + + for notebook_requirements in matching_notebook_requirements_list: + packages_common = notebook_requirements.get("packages_common", []) + packages_environment_specific = notebook_requirements.get(f"packages_{ENVIRONMENT.value}", []) + if packages_common: + packages_notebook_common.extend(packages_common) + if packages_environment_specific: + packages_notebook_environment_specific.extend(packages_environment_specific) + + # Note: environment specific packages have to be installed first, + # because in Pyodide common packages might depend on them + packages = [ + *packages_default_environment_specific, + *packages_notebook_environment_specific, + *packages_default_common, + *packages_notebook_common, + ] + return packages + + +async def install_packages_with_hashing(packages: List[str], verbose: bool = True): + """ + Install the packages listed in the requirements file for the notebook with the given name. + + Args: + notebook_name_pattern (str): The name pattern of the notebook for which to install packages. + config_file_path (str): The path to the requirements file. + verbose (bool): Whether to print the names of the installed packages and status of installation. + """ + # Hash the requirements to avoid re-installing packages + requirements_hash = str(hash(json.dumps(packages))) + if os.environ.get("requirements_hash") != requirements_hash: + for pkg in packages: + await install_package(pkg, verbose) + if verbose: + log("Packages installed successfully.", force_verbose=verbose) + os.environ["requirements_hash"] = requirements_hash + else: + if verbose: + log("Packages are already installed.", force_verbose=verbose) + + +async def install_packages(notebook_name_pattern: str, config_file_path: str = "", verbose: bool = True): + """ + Install the packages listed in the requirements file for the notebook with the given name. + + Args: + notebook_name_pattern (str): The name pattern of the notebook for which to install packages. + config_file_path (str): The path to the requirements file. + verbose (bool): Whether to print the names of the installed packages and status of installation. + """ + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + await install_init() + # PyYAML has to be installed before being imported in Pyodide and can't appear at the top of the file + import yaml + with open(get_config_yml_file_path(config_file_path), "r") as f: + requirements_dict = yaml.safe_load(f) + packages = get_packages_list(requirements_dict, notebook_name_pattern) + await install_packages_with_hashing(packages, verbose) diff --git a/src/py/mat3ra/utils/jupyterlite/settings.py b/src/py/mat3ra/utils/jupyterlite/settings.py new file mode 100644 index 0000000..7cdfe85 --- /dev/null +++ b/src/py/mat3ra/utils/jupyterlite/settings.py @@ -0,0 +1 @@ +UPLOADS_FOLDER = "uploads"