diff --git a/CHANGES.md b/CHANGES.md index 8c1508667..2b534f666 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -46,6 +46,12 @@ * xcube server can now read SNAP color palette definition files (`*.cpd`) with alpha values. (#932) +* The class `xcube.webapi.viewer.Viewer` now accepts root paths or URLs that + will each be scanned for datasets. The roots are passed as keyword argument + `roots` whose value is a path or URL or an iterable of paths or URLs. + A new keyword argument `max_depth` defines the maximum subdirectory depths + used to search for datasets in case `roots` is given. It defaults to `1`. + ### Incompatible API changes * The `get_cmap()` method of `util.cmaps.ColormapProvider` now returns a diff --git a/docs/source/api.rst b/docs/source/api.rst index 4a1140250..cf6efa724 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -208,6 +208,9 @@ Utilities .. autoclass:: xcube.core.schema.CubeSchema :members: +.. autoclass:: xcube.webapi.viewer.Viewer + :members: + .. autofunction:: xcube.util.dask.new_cluster Plugin Development diff --git a/test/webapi/viewer/test_viewer.py b/test/webapi/viewer/test_viewer.py index 7265e84ff..56c26ece6 100644 --- a/test/webapi/viewer/test_viewer.py +++ b/test/webapi/viewer/test_viewer.py @@ -4,8 +4,9 @@ import os import unittest -from typing import Optional, Any +from collections.abc import Iterable from collections.abc import Mapping +from typing import Optional, Any, Union import pytest @@ -34,8 +35,15 @@ def tearDown(self) -> None: if self.viewer is not None: self.viewer.stop_server() - def get_viewer(self, server_config: Optional[Mapping[str, Any]] = None) -> Viewer: - self.viewer = Viewer(server_config=server_config) + def get_viewer( + self, + server_config: Optional[Mapping[str, Any]] = None, + roots: Optional[Union[str, Iterable[str]]] = None, + max_depth: Optional[int] = None, + ) -> Viewer: + self.viewer = Viewer( + server_config=server_config, roots=roots, max_depth=max_depth + ) return self.viewer def test_start_and_stop_server(self): @@ -62,18 +70,73 @@ def test_show(self): def test_no_config(self): viewer = self.get_viewer() self.assertIsInstance(viewer.server_config, dict) - self.assertIn("port", viewer.server_config) - self.assertIn("address", viewer.server_config) - self.assertIn("reverse_url_prefix", viewer.server_config) + self.assertIsInstance(viewer.server_config.get("port"), int) + self.assertIsInstance(viewer.server_config.get("address"), str) + self.assertIsInstance(viewer.server_config.get("reverse_url_prefix"), str) def test_with_config(self): - viewer = self.get_viewer(STYLES_CONFIG) + viewer = self.get_viewer({"port": 8888, **STYLES_CONFIG}) self.assertIsInstance(viewer.server_config, dict) - self.assertIn("port", viewer.server_config) - self.assertIn("address", viewer.server_config) - self.assertIn("reverse_url_prefix", viewer.server_config) - self.assertIn("Styles", viewer.server_config) - self.assertEqual(STYLES_CONFIG["Styles"], viewer.server_config["Styles"]) + # Get rid of "reverse_url_prefix" as it depends on env vars + # noinspection PyUnresolvedReferences + self.assertIsInstance(viewer.server_config.pop("reverse_url_prefix", None), str) + self.assertEqual( + { + "address": "0.0.0.0", + "port": 8888, + **STYLES_CONFIG, + }, + viewer.server_config, + ) + + def test_with_root(self): + viewer = self.get_viewer({"port": 8081}, roots="data") + self.assertIsInstance(viewer.server_config, dict) + # Get rid of "reverse_url_prefix" as it depends on env vars + # noinspection PyUnresolvedReferences + self.assertIsInstance(viewer.server_config.pop("reverse_url_prefix", None), str) + self.assertEqual( + { + "address": "0.0.0.0", + "port": 8081, + "DataStores": [ + { + "Identifier": "_root_0", + "StoreId": "file", + "StoreParams": {"max_depth": 1, "root": "data"}, + } + ], + }, + viewer.server_config, + ) + + def test_with_roots(self): + viewer = self.get_viewer( + {"port": 8080}, roots=["data", "s3://xcube"], max_depth=2 + ) + self.assertIsInstance(viewer.server_config, dict) + # Get rid of "reverse_url_prefix" as it depends on env vars + # noinspection PyUnresolvedReferences + self.assertIsInstance(viewer.server_config.pop("reverse_url_prefix", None), str) + self.assertEqual( + { + "address": "0.0.0.0", + "port": 8080, + "DataStores": [ + { + "Identifier": "_root_0", + "StoreId": "file", + "StoreParams": {"max_depth": 2, "root": "data"}, + }, + { + "Identifier": "_root_1", + "StoreId": "s3", + "StoreParams": {"max_depth": 2, "root": "xcube"}, + }, + ], + }, + viewer.server_config, + ) def test_urls(self): viewer = self.get_viewer() diff --git a/xcube/webapi/viewer/viewer.py b/xcube/webapi/viewer/viewer.py index 954040f11..2e45b5389 100644 --- a/xcube/webapi/viewer/viewer.py +++ b/xcube/webapi/viewer/viewer.py @@ -8,8 +8,10 @@ import threading from pathlib import Path from typing import Optional, Union, Any, Tuple, Dict +from collections.abc import Iterable from collections.abc import Mapping +import fsspec import tornado.ioloop import xarray as xr @@ -30,33 +32,59 @@ _LAB_INFO_FILE = "~/.xcube/jupyterlab/lab-info.json" +_DEFAULT_MAX_DEPTH = 1 + + class Viewer: - """Experimental class that represents the xcube Viewer - in Jupyter Notebooks. + """xcube Viewer for Jupyter Notebooks. - Args: - server_config: Server configuration. See "xcube serve --show - configschema". - """ + The viewer can be used to visualise and inspect datacubes + with at least one data variable with dimensions ``["time", "lat", "lon"]`` + or, if a grid mapping is present, with arbitrary ``"time"`` and + arbitrarily x- and y-dimensions, e.g., ``["time", "y", "x"]`` . - def __init__(self, server_config: Optional[Mapping[str, Any]] = None): - server_config = dict(server_config or {}) + Add datacubes from instances of ``xarray.Dataset``: - port = server_config.get("port") - address = server_config.get("address") + ``` + viewer = Viewer() + viewer.add_dataset(dataset) # can set color styles here too, see doc below + viewer.show() + ``` - if port is None: - port = _find_port() - if address is None: - address = "0.0.0.0" + Display all datasets of formats Zarr, NetCDF, COG/GeoTIFF found in the + given directories in the local filesystem or in a given S3 bucket: - server_config["port"] = port - server_config["address"] = address + ``` + viewer = Viewer(roots=["/eodata/smos/l2", "s3://xcube/examples"]) + viewer.show() + ``` - server_url, reverse_url_prefix = _get_server_url_and_rev_prefix(port) - server_config["reverse_url_prefix"] = reverse_url_prefix + The `Viewer` class takes a xcube server configuration as first + argument. More details regarding configuration parameters are given in the + `server documentation `_. + The full configuration reference can be generated by excecuting CLI command + ``$ xcube serve --show configschema``. - self._server_config = server_config + Args: + server_config: Server configuration. + See also output of ``$ xcube serve --show configschema``. + roots: A path or URL or an iterable of paths or URLs + that will each be scanned for datasets to be shown in the viewer. + max_depth: defines the maximum subdirectory depth used to + search for datasets in case *roots* is given. + """ + + def __init__( + self, + server_config: Optional[Mapping[str, Any]] = None, + roots: Optional[Union[str, Iterable[str]]] = None, + max_depth: Optional[int] = None, + ): + self._server_config, server_url = _get_server_config( + server_config=server_config, roots=roots, max_depth=max_depth + ) + self._server_url = server_url + self._viewer_url = f"{server_url}/viewer/?serverUrl={server_url}" # Got trick from # https://stackoverflow.com/questions/55201748/running-a-tornado-server-within-a-jupyter-notebook @@ -67,14 +95,11 @@ def __init__(self, server_config: Optional[Mapping[str, Any]] = None): self._server = Server( TornadoFramework(io_loop=self._io_loop, shared_io_loop=True), - config=server_config, + config=self._server_config, ) self._io_loop.add_callback(self._server.start) - self._server_url = server_url - self._viewer_url = f"{server_url}/viewer/?serverUrl={server_url}" - @property def server_config(self) -> Mapping[str, Any]: """The server configuration used by this viewer.""" @@ -171,6 +196,7 @@ def show(self, width: Union[int, str] = "100%", height: Union[str, int] = 800): height: The height of the viewer's iframe. """ try: + # noinspection PyPackageRequirements from IPython.core.display import HTML return HTML( @@ -201,6 +227,37 @@ def _check_server_running(self): return self.is_server_running +def _get_server_config( + server_config: Optional[Mapping[str, Any]] = None, + roots: Optional[Union[str, Iterable[str]]] = None, + max_depth: Optional[int] = None, +) -> tuple[dict[str, Any], str]: + server_config = dict(server_config or {}) + max_depth = max_depth or _DEFAULT_MAX_DEPTH + + port = server_config.get("port") + address = server_config.get("address") + + if port is None: + port = _find_port() + if address is None: + address = "0.0.0.0" + + server_config["port"] = port + server_config["address"] = address + + server_url, reverse_url_prefix = _get_server_url_and_rev_prefix(port) + server_config["reverse_url_prefix"] = reverse_url_prefix + + if roots is not None: + roots = [roots] if isinstance(roots, str) else roots + config_stores = list(server_config.get("DataStores", [])) + root_stores = _get_data_stores_from_roots(roots, max_depth) + server_config["DataStores"] = config_stores + root_stores + + return server_config, server_url + + def _get_server_url_and_rev_prefix(port: int) -> tuple[str, str]: lab_url = os.environ.get(_LAB_URL_ENV_VAR) or None has_proxy = lab_url is not None @@ -234,3 +291,19 @@ def _find_port(start: int = 8000, end: Optional[int] = None) -> int: if s.connect_ex(("localhost", port)) != 0: return port raise RuntimeError("No available port found") + + +def _get_data_stores_from_roots( + roots: Iterable[str], max_depth: int +) -> list[dict[str, dict]]: + extra_data_stores = [] + for index, root in enumerate(roots): + protocol, path = fsspec.core.split_protocol(root) + extra_data_stores.append( + { + "Identifier": f"_root_{index}", + "StoreId": protocol or "file", + "StoreParams": {"root": path, "max_depth": max_depth}, + } + ) + return extra_data_stores