Merge pull request #68 from dcs4cop/forman-serve_wo_config

allow "xcube serve" to serve cubes given via CLI paths

CHANGES.md

@@ -2,6 +2,9 @@
 
 * Restructured and clarified code base (#27)
 * Moved to Python 3.7 (#25)
+* `xcube serve` can now be run with data cube paths and styling information given via the CLI rather 
+  than a configuration file. For example `xcube serve --styles conc_chl=(0,20,"viridis") /path/to/my/chl-cube.zarr`.
+  This allows for quick inspection of newly generated cubes via `xcube gen`.
 * Added global `xcube --scheduler <scheduler>` option for Dask distributed computing (#58)
 * Added global `xcube --traceback` option, removed local `xcube gen --traceback` option
 * Completed version 1 of an xcube developer guide.

docs/XCUBE-SPEC.md

@@ -76,6 +76,16 @@ ARD links:
     * `[time, ..., lat, lon]` (see WGS84 schema) or 
     * `[time, ..., y, x]` (see Generic schema) 
   * MAY have extra dimensions, e.g. `layer` (of the atmosphere), `band` (of a spectrum).
+  * SHALL specify the `units` metadata attribute.
+  * SHOULD specify metadata attributes that are used to identify missing values, namely
+    `_FillValue` and / or `valid_min`, `valid_max`, see notes in CF conventions on these attributes.
+  * MAY specify metadata attributes that can be used to visualise the data:
+    * `color_bar_name`: Name of a predefined colour mapping. The colour bar is applied
+       between a minimum and a maximum value. 
+    * `color_value_min`, `color_value_max`: Minimum and maximum value for applying the colour bar.
+       If not provided, minimum and maximum default to `valid_min`, `valid_max`. If neither 
+       these are provided, minimum and maximum default to `0` and `1`.
+
 
 
 ### WGS84 Schema (extends Basic)

test/util/test_cliutil.py

@@ -12,6 +12,9 @@ def test_parse_cli_kwargs(self):
                          parse_cli_kwargs("", metavar="<chunks>"))
         self.assertEqual(dict(time=1, lat=256, lon=512),
                          parse_cli_kwargs("time=1, lat=256, lon=512", metavar="<chunks>"))
+        self.assertEqual(dict(chl_conc=(0, 20, 'greens'), chl_tsm=(0, 15, 'viridis')),
+                         parse_cli_kwargs("chl_conc=(0,20,'greens'),chl_tsm=(0,15,'viridis')",
+                                          metavar="<styles>"))
 
         with self.assertRaises(click.ClickException) as cm:
             parse_cli_kwargs("45 * 'A'", metavar="<chunks>")

test/webapi/test_service.py

@@ -1,8 +1,7 @@
 import re
 import unittest
 
-from xcube.webapi import service
-from xcube.webapi.service import parse_tile_cache_config
+from xcube.webapi.service import url_pattern, parse_tile_cache_config, new_default_config
 
 
 class TileCacheConfigTest(unittest.TestCase):
@@ -30,39 +29,80 @@ def test_parse_tile_cache_config(self):
         self.assertEqual("negative tile cache size: '-2G'", f"{cm.exception}")
 
 
+class DefaultConfigTest(unittest.TestCase):
+    def test_new_default_config(self):
+        config = new_default_config(["/home/bibo/data/cube-1.zarr",
+                                     "/home/bibo/data/cube-2.nc"],
+                                    dict(conc_chl=(0.0, 20.0),
+                                         conc_tsm=(0.0, 12.0, 'plasma')))
+        self.assertEqual({
+            'Datasets': [
+                {
+                    'FileSystem': 'local',
+                    'Format': 'zarr',
+                    'Identifier': 'dataset_1',
+                    'Path': '/home/bibo/data/cube-1.zarr',
+                    'Title': 'Dataset #1'
+                },
+                {
+                    'FileSystem': 'local',
+                    'Format': 'netcdf4',
+                    'Identifier': 'dataset_2',
+                    'Path': '/home/bibo/data/cube-2.nc',
+                    'Title': 'Dataset #2'
+                }
+            ],
+            'Styles': [
+                {'Identifier': 'default',
+                 'ColorMappings': {
+                     'conc_chl': {'ValueRange': [0.0, 20.0]},
+                     'conc_tsm': {'ColorBar': 'plasma',
+                                  'ValueRange': [0.0, 12.0]}},
+                 }
+            ]},
+            config)
+
+        with self.assertRaises(ValueError) as cm:
+            new_default_config(["/home/bibo/data/cube-1.zarr",
+                                "/home/bibo/data/cube-2.nc"],
+                               dict(conc_chl=20.0,
+                                    conc_tsm=(0.0, 12.0, 'plasma')))
+        self.assertEqual("illegal style: conc_chl=20.0", f"{cm.exception}")
+
+
 class UrlPatternTest(unittest.TestCase):
     def test_url_pattern_works(self):
-        re_pattern = service.url_pattern('/open/{{id1}}ws/{{id2}}wf')
+        re_pattern = url_pattern('/open/{{id1}}ws/{{id2}}wf')
         matcher = re.fullmatch(re_pattern, '/open/34ws/a66wf')
         self.assertIsNotNone(matcher)
         self.assertEqual(matcher.groupdict(), {'id1': '34', 'id2': 'a66'})
 
-        re_pattern = service.url_pattern('/open/ws{{id1}}/wf{{id2}}')
+        re_pattern = url_pattern('/open/ws{{id1}}/wf{{id2}}')
         matcher = re.fullmatch(re_pattern, '/open/ws34/wfa66')
         self.assertIsNotNone(matcher)
         self.assertEqual(matcher.groupdict(), {'id1': '34', 'id2': 'a66'})
 
         x = 'C%3A%5CUsers%5CNorman%5CIdeaProjects%5Cccitools%5Cect-core%5Ctest%5Cui%5CTEST_WS_3'
-        re_pattern = service.url_pattern('/ws/{{base_dir}}/res/{{res_name}}/add')
+        re_pattern = url_pattern('/ws/{{base_dir}}/res/{{res_name}}/add')
         matcher = re.fullmatch(re_pattern, '/ws/%s/res/SST/add' % x)
         self.assertIsNotNone(matcher)
         self.assertEqual(matcher.groupdict(), {'base_dir': x, 'res_name': 'SST'})
 
     def test_url_pattern_ok(self):
-        self.assertEqual(service.url_pattern('/version'),
+        self.assertEqual(url_pattern('/version'),
                          '/version')
-        self.assertEqual(service.url_pattern('{{num}}/get'),
+        self.assertEqual(url_pattern('{{num}}/get'),
                          '(?P<num>[^\;\/\?\:\@\&\=\+\$\,]+)/get')
-        self.assertEqual(service.url_pattern('/open/{{ws_name}}'),
+        self.assertEqual(url_pattern('/open/{{ws_name}}'),
                          '/open/(?P<ws_name>[^\;\/\?\:\@\&\=\+\$\,]+)')
-        self.assertEqual(service.url_pattern('/open/ws{{id1}}/wf{{id2}}'),
+        self.assertEqual(url_pattern('/open/ws{{id1}}/wf{{id2}}'),
                          '/open/ws(?P<id1>[^\;\/\?\:\@\&\=\+\$\,]+)/wf(?P<id2>[^\;\/\?\:\@\&\=\+\$\,]+)')
 
     def test_url_pattern_fail(self):
         with self.assertRaises(ValueError) as cm:
-            service.url_pattern('/open/{{ws/name}}')
+            url_pattern('/open/{{ws/name}}')
         self.assertEqual(str(cm.exception), 'name in {{name}} must be a valid identifier, but got "ws/name"')
 
         with self.assertRaises(ValueError) as cm:
-            service.url_pattern('/info/{{id}')
+            url_pattern('/info/{{id}')
         self.assertEqual(str(cm.exception), 'no matching "}}" after "{{" in "/info/{{id}"')

xcube/cli/serve.py

@@ -18,17 +18,20 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+from typing import List
 
 import click
 
+from xcube.util.cliutil import parse_cli_kwargs
 from xcube.webapi import __version__, __description__
 from xcube.webapi.defaults import DEFAULT_PORT, DEFAULT_NAME, DEFAULT_ADDRESS, DEFAULT_UPDATE_PERIOD, \
-    DEFAULT_CONFIG_FILE, DEFAULT_TILE_CACHE_SIZE, DEFAULT_TILE_COMP_MODE
+    DEFAULT_TILE_CACHE_SIZE, DEFAULT_TILE_COMP_MODE
 
 __author__ = "Norman Fomferra (Brockmann Consult GmbH)"
 
 
 @click.command(name='serve')
+@click.argument('cubes', metavar='CUBE...', nargs=-1)
 @click.version_option(__version__)
 @click.option('--name', '-n', metavar='NAME', default=DEFAULT_NAME,
               help=f'Service name. Defaults to {DEFAULT_NAME!r}.')
@@ -41,9 +44,14 @@
               help='Service will update after given seconds of inactivity. Zero or a negative value will '
                    'disable update checks. '
                    f'Defaults to {DEFAULT_UPDATE_PERIOD!r}.')
-@click.option('--config', '-c', metavar='FILE', default=None,
-              help='Datasets configuration file. '
-                   f'Defaults to {DEFAULT_CONFIG_FILE!r}.')
+@click.option('--styles', '-s', metavar='STYLES', default=None,
+              help='Color mapping styles for variables. '
+                   'Used only, if one or more CUBE arguments are provided and CONFIG is not given. '
+                   'Comma-separated list with elements of the form '
+                   '<var>=(<vmin>,<vmax>) or <var>=(<vmin>,<vmax>,"<cmap>")')
+@click.option('--config', '-c', metavar='CONFIG', default=None,
+              help='Use datasets configuration file CONFIG. '
+                   'Cannot be used if CUBES are provided.')
 @click.option('--tilecache', metavar='SIZE', default=DEFAULT_TILE_CACHE_SIZE,
               help=f'In-memory tile cache size in bytes. '
                    f'Unit suffixes {"K"!r}, {"M"!r}, {"G"!r} may be used. '
@@ -57,10 +65,12 @@
               help="Delegate logging to the console (stderr).")
 @click.option('--traceperf', is_flag=True,
               help="Print performance diagnostics (stdout).")
-def serve(name: str,
+def serve(cubes: List[str],
+          name: str,
           address: str,
           port: int,
           update: float,
+          styles: str,
           config: str,
           tilecache: str,
           tilemode: int,
@@ -73,26 +83,28 @@ def serve(name: str,
     The RESTful API documentation can be found at https://app.swaggerhub.com/apis/bcdev/xcube-server.
     """
 
+    if config and cubes:
+        raise click.ClickException("CONFIG and CUBES cannot be used at the same time.")
+    if styles:
+        styles = parse_cli_kwargs(styles, "STYLES")
     from xcube.webapi.app import new_application
     from xcube.webapi.service import Service
 
-    try:
-        print(f'{__description__}, version {__version__}')
-        service = Service(new_application(name),
-                          name=name,
-                          port=port,
-                          address=address,
-                          config_file=config,
-                          tile_cache_size=tilecache,
-                          tile_comp_mode=tilemode,
-                          update_period=update,
-                          log_to_stderr=verbose,
-                          trace_perf=traceperf)
-        service.start()
-        return 0
-    except Exception as e:
-        print('error: %s' % e)
-        return 1
+    print(f'{__description__}, version {__version__}')
+    service = Service(new_application(name),
+                      name=name,
+                      port=port,
+                      address=address,
+                      cube_paths=cubes,
+                      styles=styles,
+                      config_file=config,
+                      tile_cache_size=tilecache,
+                      tile_comp_mode=tilemode,
+                      update_period=update,
+                      log_to_stderr=verbose,
+                      trace_perf=traceperf)
+    service.start()
+    return 0
 
 
 def main(args=None):

xcube/webapi/context.py

@@ -40,8 +40,11 @@
 from .mldataset import FileStorageMultiLevelDataset, BaseMultiLevelDataset, MultiLevelDataset, \
     ComputedMultiLevelDataset, ObjectStorageMultiLevelDataset
 from .reqparams import RequestParams
+from ..util.dsio import guess_dataset_format, FORMAT_NAME_NETCDF4, FORMAT_NAME_ZARR
 from ..util.perf import measure_time
 
+FORMAT_NAME_LEVELS = 'levels'
+
 COMPUTE_DATASET = 'compute_dataset'
 ALL_PLACES = "all"
 
@@ -184,6 +187,7 @@ def get_tile_grid(self, ds_id: str) -> TileGrid:
         return ml_dataset.tile_grid
 
     def get_color_mapping(self, ds_id: str, var_name: str):
+        cmap_cbar, cmap_vmin, cmap_vmax = DEFAULT_CMAP_CBAR, DEFAULT_CMAP_VMIN, DEFAULT_CMAP_VMAX
         dataset_descriptor = self.get_dataset_descriptor(ds_id)
         style_name = dataset_descriptor.get('Style', 'default')
         styles = self._config.get('Styles')
@@ -192,18 +196,25 @@ def get_color_mapping(self, ds_id: str, var_name: str):
             for s in styles:
                 if style_name == s['Identifier']:
                     style = s
+                    break
             # TODO: check color_mappings is not None
             if style:
                 color_mappings = style.get('ColorMappings')
                 if color_mappings:
                     # TODO: check color_mappings is not None
                     color_mapping = color_mappings.get(var_name)
                     if color_mapping:
-                        cmap_cbar = color_mapping.get('ColorBar', DEFAULT_CMAP_CBAR)
-                        cmap_vmin, cmap_vmax = color_mapping.get('ValueRange', (DEFAULT_CMAP_VMIN, DEFAULT_CMAP_VMAX))
+                        cmap_cbar = color_mapping.get('ColorBar', cmap_cbar)
+                        cmap_vmin, cmap_vmax = color_mapping.get('ValueRange', (cmap_vmin, cmap_vmax))
                         return cmap_cbar, cmap_vmin, cmap_vmax
+            else:
+                ds, var = self.get_dataset_and_variable(ds_id, var_name)
+                cmap_cbar = var.attrs.get('color_bar_name', cmap_cbar)
+                cmap_vmin = var.attrs.get('color_value_min', cmap_vmin)
+                cmap_vmax = var.attrs.get('color_value_max', cmap_vmax)
+
         _LOG.warning(f'color mapping for variable {var_name!r} of dataset {ds_id!r} undefined: using defaults')
-        return DEFAULT_CMAP_CBAR, DEFAULT_CMAP_VMIN, DEFAULT_CMAP_VMAX
+        return cmap_cbar, cmap_vmin, cmap_vmax
 
     def _get_dataset_entry(self, ds_id: str) -> Tuple[MultiLevelDataset, Dict[str, Any]]:
         if ds_id not in self._dataset_cache:
@@ -399,6 +410,12 @@ def find_dataset_descriptor(cls,
         return next((dsd for dsd in dataset_descriptors if dsd['Identifier'] == ds_name), None)
 
 
+def guess_cube_format(path: str) -> str:
+    if path.endswith('.levels'):
+        return FORMAT_NAME_LEVELS
+    return guess_dataset_format(path)
+
+
 # noinspection PyUnusedLocal
 def open_ml_dataset_from_object_storage(ctx: ServiceContext,
                                         dataset_descriptor: DatasetDescriptor) -> MultiLevelDataset:
@@ -408,7 +425,7 @@ def open_ml_dataset_from_object_storage(ctx: ServiceContext,
     if not path:
         raise ServiceConfigError(f"Missing 'path' entry in dataset descriptor {ds_id}")
 
-    data_format = dataset_descriptor.get('Format', 'zarr')
+    data_format = dataset_descriptor.get('Format', FORMAT_NAME_ZARR)
 
     s3_client_kwargs = {}
     if 'Endpoint' in dataset_descriptor:
@@ -417,14 +434,14 @@ def open_ml_dataset_from_object_storage(ctx: ServiceContext,
         s3_client_kwargs['region_name'] = dataset_descriptor['Region']
     obs_file_system = s3fs.S3FileSystem(anon=True, client_kwargs=s3_client_kwargs)
 
-    if data_format == 'zarr':
+    if data_format == FORMAT_NAME_ZARR:
         store = s3fs.S3Map(root=path, s3=obs_file_system, check=False)
         cached_store = zarr.LRUStoreCache(store, max_size=2 ** 28)
         with measure_time(tag=f"opened remote zarr dataset {path}"):
             ds = xr.open_zarr(cached_store)
         return BaseMultiLevelDataset(ds)
 
-    if data_format == 'levels':
+    if data_format == FORMAT_NAME_LEVELS:
         with measure_time(tag=f"opened remote levels dataset {path}"):
             return ObjectStorageMultiLevelDataset(ds_id, obs_file_system, path,
                                                   exception_type=ServiceConfigError)
@@ -440,22 +457,24 @@ def open_ml_dataset_from_local_fs(ctx: ServiceContext, dataset_descriptor: Datas
     if not os.path.isabs(path):
         path = os.path.join(ctx.base_dir, path)
 
-    data_format = dataset_descriptor.get('Format', 'nc')
+    data_format = dataset_descriptor.get('Format', guess_cube_format(path))
 
-    if data_format == 'nc':
+    if data_format == FORMAT_NAME_NETCDF4:
         with measure_time(tag=f"opened local NetCDF dataset {path}"):
             ds = xr.open_dataset(path)
             return BaseMultiLevelDataset(ds)
 
-    if data_format == 'zarr':
+    if data_format == FORMAT_NAME_ZARR:
         with measure_time(tag=f"opened local zarr dataset {path}"):
             ds = xr.open_zarr(path)
             return BaseMultiLevelDataset(ds)
 
-    if data_format == 'levels':
+    if data_format == FORMAT_NAME_LEVELS:
         with measure_time(tag=f"opened local levels dataset {path}"):
             return FileStorageMultiLevelDataset(path)
 
+    raise ServiceConfigError(f"Illegal data format {data_format!r} for dataset {ds_id}")
+
 
 def open_ml_dataset_from_python_code(ctx: ServiceContext, dataset_descriptor: DatasetDescriptor) -> MultiLevelDataset:
     ds_id = dataset_descriptor.get('Identifier')

xcube/webapi/defaults.py

@@ -26,7 +26,6 @@
 DEFAULT_NAME = 'xcube'
 DEFAULT_ADDRESS = 'localhost'
 DEFAULT_PORT = 8080
-DEFAULT_CONFIG_FILE = os.path.abspath('xcube_server.yml')
 DEFAULT_TILE_CACHE_SIZE = "512M"
 DEFAULT_UPDATE_PERIOD = 2.
 DEFAULT_LOG_PREFIX = os.path.abspath('xcube_server.log')