Allow passing roots to xcube.webapi.viewer.Viewer

CHANGES.md

@@ -46,6 +46,12 @@
 * xcube server can now read SNAP color palette definition files (`*.cpd`) with
   alpha values. (#932)
 
+* The class `xcube.webapi.viewer.Viewer` now accepts root paths or URLs that 
+  will each be scanned for datasets. The roots are passed as keyword argument
+  `roots` whose value is a path or URL or an iterable of paths or URLs. 
+  A new keyword argument `max_depth` defines the maximum subdirectory depths 
+  used to search for datasets in case `roots` is given. It defaults to `1`.
+
 ### Incompatible API changes
 
 * The `get_cmap()` method of `util.cmaps.ColormapProvider` now returns a 

docs/source/api.rst

@@ -208,6 +208,9 @@ Utilities
 .. autoclass:: xcube.core.schema.CubeSchema
     :members:
 
+.. autoclass:: xcube.webapi.viewer.Viewer
+    :members:
+
 .. autofunction:: xcube.util.dask.new_cluster
 
 Plugin Development

test/webapi/viewer/test_viewer.py

@@ -4,8 +4,9 @@
 
 import os
 import unittest
-from typing import Optional, Any
+from collections.abc import Iterable
 from collections.abc import Mapping
+from typing import Optional, Any, Union
 
 import pytest
 
@@ -34,8 +35,15 @@ def tearDown(self) -> None:
         if self.viewer is not None:
             self.viewer.stop_server()
 
-    def get_viewer(self, server_config: Optional[Mapping[str, Any]] = None) -> Viewer:
-        self.viewer = Viewer(server_config=server_config)
+    def get_viewer(
+        self,
+        server_config: Optional[Mapping[str, Any]] = None,
+        roots: Optional[Union[str, Iterable[str]]] = None,
+        max_depth: Optional[int] = None,
+    ) -> Viewer:
+        self.viewer = Viewer(
+            server_config=server_config, roots=roots, max_depth=max_depth
+        )
         return self.viewer
 
     def test_start_and_stop_server(self):
@@ -62,18 +70,73 @@ def test_show(self):
     def test_no_config(self):
         viewer = self.get_viewer()
         self.assertIsInstance(viewer.server_config, dict)
-        self.assertIn("port", viewer.server_config)
-        self.assertIn("address", viewer.server_config)
-        self.assertIn("reverse_url_prefix", viewer.server_config)
+        self.assertIsInstance(viewer.server_config.get("port"), int)
+        self.assertIsInstance(viewer.server_config.get("address"), str)
+        self.assertIsInstance(viewer.server_config.get("reverse_url_prefix"), str)
 
     def test_with_config(self):
-        viewer = self.get_viewer(STYLES_CONFIG)
+        viewer = self.get_viewer({"port": 8888, **STYLES_CONFIG})
         self.assertIsInstance(viewer.server_config, dict)
-        self.assertIn("port", viewer.server_config)
-        self.assertIn("address", viewer.server_config)
-        self.assertIn("reverse_url_prefix", viewer.server_config)
-        self.assertIn("Styles", viewer.server_config)
-        self.assertEqual(STYLES_CONFIG["Styles"], viewer.server_config["Styles"])
+        # Get rid of "reverse_url_prefix" as it depends on env vars
+        # noinspection PyUnresolvedReferences
+        self.assertIsInstance(viewer.server_config.pop("reverse_url_prefix", None), str)
+        self.assertEqual(
+            {
+                "address": "0.0.0.0",
+                "port": 8888,
+                **STYLES_CONFIG,
+            },
+            viewer.server_config,
+        )
+
+    def test_with_root(self):
+        viewer = self.get_viewer({"port": 8081}, roots="data")
+        self.assertIsInstance(viewer.server_config, dict)
+        # Get rid of "reverse_url_prefix" as it depends on env vars
+        # noinspection PyUnresolvedReferences
+        self.assertIsInstance(viewer.server_config.pop("reverse_url_prefix", None), str)
+        self.assertEqual(
+            {
+                "address": "0.0.0.0",
+                "port": 8081,
+                "DataStores": [
+                    {
+                        "Identifier": "_root_0",
+                        "StoreId": "file",
+                        "StoreParams": {"max_depth": 1, "root": "data"},
+                    }
+                ],
+            },
+            viewer.server_config,
+        )
+
+    def test_with_roots(self):
+        viewer = self.get_viewer(
+            {"port": 8080}, roots=["data", "s3://xcube"], max_depth=2
+        )
+        self.assertIsInstance(viewer.server_config, dict)
+        # Get rid of "reverse_url_prefix" as it depends on env vars
+        # noinspection PyUnresolvedReferences
+        self.assertIsInstance(viewer.server_config.pop("reverse_url_prefix", None), str)
+        self.assertEqual(
+            {
+                "address": "0.0.0.0",
+                "port": 8080,
+                "DataStores": [
+                    {
+                        "Identifier": "_root_0",
+                        "StoreId": "file",
+                        "StoreParams": {"max_depth": 2, "root": "data"},
+                    },
+                    {
+                        "Identifier": "_root_1",
+                        "StoreId": "s3",
+                        "StoreParams": {"max_depth": 2, "root": "xcube"},
+                    },
+                ],
+            },
+            viewer.server_config,
+        )
 
     def test_urls(self):
         viewer = self.get_viewer()

xcube/webapi/viewer/viewer.py

@@ -8,8 +8,10 @@
 import threading
 from pathlib import Path
 from typing import Optional, Union, Any, Tuple, Dict
+from collections.abc import Iterable
 from collections.abc import Mapping
 
+import fsspec
 import tornado.ioloop
 import xarray as xr
 
@@ -30,33 +32,59 @@
 _LAB_INFO_FILE = "~/.xcube/jupyterlab/lab-info.json"
 
 
+_DEFAULT_MAX_DEPTH = 1
+
+
 class Viewer:
-    """Experimental class that represents the xcube Viewer
-    in Jupyter Notebooks.
+    """xcube Viewer for Jupyter Notebooks.
 
-    Args:
-        server_config: Server configuration. See "xcube serve --show
-            configschema".
-    """
+    The viewer can be used to visualise and inspect datacubes
+    with at least one data variable with dimensions ``["time", "lat", "lon"]``
+    or, if a grid mapping is present, with arbitrary ``"time"`` and
+    arbitrarily x- and y-dimensions, e.g., ``["time", "y", "x"]`` .
 
-    def __init__(self, server_config: Optional[Mapping[str, Any]] = None):
-        server_config = dict(server_config or {})
+    Add datacubes from instances of ``xarray.Dataset``:
 
-        port = server_config.get("port")
-        address = server_config.get("address")
+    ```
+    viewer = Viewer()
+    viewer.add_dataset(dataset)  # can set color styles here too, see doc below
+    viewer.show()
+    ```
 
-        if port is None:
-            port = _find_port()
-        if address is None:
-            address = "0.0.0.0"
+    Display all datasets of formats Zarr, NetCDF, COG/GeoTIFF found in the
+    given directories in the local filesystem or in a given S3 bucket:
 
-        server_config["port"] = port
-        server_config["address"] = address
+    ```
+    viewer = Viewer(roots=["/eodata/smos/l2", "s3://xcube/examples"])
+    viewer.show()
+    ```
 
-        server_url, reverse_url_prefix = _get_server_url_and_rev_prefix(port)
-        server_config["reverse_url_prefix"] = reverse_url_prefix
+    The `Viewer` class takes a xcube server configuration as first
+    argument. More details regarding configuration parameters are given in the
+    `server documentation <https://xcube.readthedocs.io/en/latest/cli/xcube_serve.html>`_.
+    The full configuration reference can be generated by excecuting CLI command
+    ``$ xcube serve --show configschema``.
 
-        self._server_config = server_config
+    Args:
+        server_config: Server configuration.
+            See also output of ``$ xcube serve --show configschema``.
+        roots: A path or URL or an iterable of paths or URLs
+            that will each be scanned for datasets to be shown in the viewer.
+        max_depth: defines the maximum subdirectory depth used to
+            search for datasets in case *roots* is given.
+    """
+
+    def __init__(
+        self,
+        server_config: Optional[Mapping[str, Any]] = None,
+        roots: Optional[Union[str, Iterable[str]]] = None,
+        max_depth: Optional[int] = None,
+    ):
+        self._server_config, server_url = _get_server_config(
+            server_config=server_config, roots=roots, max_depth=max_depth
+        )
+        self._server_url = server_url
+        self._viewer_url = f"{server_url}/viewer/?serverUrl={server_url}"
 
         # Got trick from
         # https://stackoverflow.com/questions/55201748/running-a-tornado-server-within-a-jupyter-notebook
@@ -67,14 +95,11 @@ def __init__(self, server_config: Optional[Mapping[str, Any]] = None):
 
         self._server = Server(
             TornadoFramework(io_loop=self._io_loop, shared_io_loop=True),
-            config=server_config,
+            config=self._server_config,
         )
 
         self._io_loop.add_callback(self._server.start)
 
-        self._server_url = server_url
-        self._viewer_url = f"{server_url}/viewer/?serverUrl={server_url}"
-
     @property
     def server_config(self) -> Mapping[str, Any]:
         """The server configuration used by this viewer."""
@@ -171,6 +196,7 @@ def show(self, width: Union[int, str] = "100%", height: Union[str, int] = 800):
             height: The height of the viewer's iframe.
         """
         try:
+            # noinspection PyPackageRequirements
             from IPython.core.display import HTML
 
             return HTML(
@@ -201,6 +227,37 @@ def _check_server_running(self):
         return self.is_server_running
 
 
+def _get_server_config(
+    server_config: Optional[Mapping[str, Any]] = None,
+    roots: Optional[Union[str, Iterable[str]]] = None,
+    max_depth: Optional[int] = None,
+) -> tuple[dict[str, Any], str]:
+    server_config = dict(server_config or {})
+    max_depth = max_depth or _DEFAULT_MAX_DEPTH
+
+    port = server_config.get("port")
+    address = server_config.get("address")
+
+    if port is None:
+        port = _find_port()
+    if address is None:
+        address = "0.0.0.0"
+
+    server_config["port"] = port
+    server_config["address"] = address
+
+    server_url, reverse_url_prefix = _get_server_url_and_rev_prefix(port)
+    server_config["reverse_url_prefix"] = reverse_url_prefix
+
+    if roots is not None:
+        roots = [roots] if isinstance(roots, str) else roots
+        config_stores = list(server_config.get("DataStores", []))
+        root_stores = _get_data_stores_from_roots(roots, max_depth)
+        server_config["DataStores"] = config_stores + root_stores
+
+    return server_config, server_url
+
+
 def _get_server_url_and_rev_prefix(port: int) -> tuple[str, str]:
     lab_url = os.environ.get(_LAB_URL_ENV_VAR) or None
     has_proxy = lab_url is not None
@@ -234,3 +291,19 @@ def _find_port(start: int = 8000, end: Optional[int] = None) -> int:
             if s.connect_ex(("localhost", port)) != 0:
                 return port
     raise RuntimeError("No available port found")
+
+
+def _get_data_stores_from_roots(
+    roots: Iterable[str], max_depth: int
+) -> list[dict[str, dict]]:
+    extra_data_stores = []
+    for index, root in enumerate(roots):
+        protocol, path = fsspec.core.split_protocol(root)
+        extra_data_stores.append(
+            {
+                "Identifier": f"_root_{index}",
+                "StoreId": protocol or "file",
+                "StoreParams": {"root": path, "max_depth": max_depth},
+            }
+        )
+    return extra_data_stores