Resolve relative links used by multi-level datasets as specified

CHANGES.md

@@ -29,8 +29,11 @@
     If given, it forces the spatial dimensions to use the specified 
     chunking.
   - Added a new example notebook 
-    [5_multi_level_datasets.ipynb](./examples/notebooks/datastores/5_multi_level_datasets.ipynb) 
-    that demonstrates the effect of the new parameters.
+    [5_multi_level_datasets.ipynb](https://github.com/dcs4cop/xcube/blob/master/examples/notebooks/datastores/5_multi_level_datasets.ipynb) 
+    that demonstrates writing and opening multi-level datasets with the 
+    xcube filesystem data stores.
+  - Specified [xcube Multi-Resolution Datasets](https://github.com/dcs4cop/xcube/blob/master/docs/source/mldatasets.md)
+    definition and format.
 
 * `xcube gen2` returns more expressive error messages.
 

test/core/store/fs/test_helpers.py

@@ -0,0 +1,61 @@
+import pathlib
+import unittest
+
+import fsspec
+import fsspec.implementations.local
+
+from xcube.core.store.fs.helpers import get_fs_path_class
+from xcube.core.store.fs.helpers import resolve_path
+
+
+class FsHelpersTest(unittest.TestCase):
+
+    def test_resolve_path_succeeds(self):
+        path = pathlib.PurePosixPath("a", "b", "c")
+        self.assertEqual(pathlib.PurePosixPath("a", "b", "c"),
+                         resolve_path(path))
+
+        path = pathlib.PurePosixPath(".", "b", "c")
+        self.assertEqual(pathlib.PurePosixPath("b", "c"),
+                         resolve_path(path))
+
+        path = pathlib.PurePosixPath("a", ".", "c")
+        self.assertEqual(pathlib.PurePosixPath("a", "c"),
+                         resolve_path(path))
+
+        path = pathlib.PurePosixPath("a", "b", ".")
+        self.assertEqual(pathlib.PurePosixPath("a", "b"),
+                         resolve_path(path))
+
+        path = pathlib.PurePosixPath("a", "..", "c")
+        self.assertEqual(pathlib.PurePosixPath("c"),
+                         resolve_path(path))
+
+        path = pathlib.PurePosixPath("a", "b", "..")
+        self.assertEqual(pathlib.PurePosixPath("a"),
+                         resolve_path(path))
+
+    def test_resolve_path_fails(self):
+        path = pathlib.PurePosixPath("..", "b", "c")
+        with self.assertRaises(ValueError) as cm:
+            resolve_path(path)
+        self.assertEqual('cannot resolve path, misplaced ".."',
+                         f'{cm.exception}')
+
+        path = pathlib.PurePosixPath("a", "..", "..")
+        with self.assertRaises(ValueError) as cm:
+            resolve_path(path)
+        self.assertEqual('cannot resolve path, misplaced ".."',
+                         f'{cm.exception}')
+
+    def test_get_fs_path_class(self):
+        self.assertIs(pathlib.PurePath,
+                      get_fs_path_class(
+                          fsspec.implementations.local.LocalFileSystem()
+                      ))
+        self.assertIs(pathlib.PurePath,
+                      get_fs_path_class(fsspec.filesystem("file")))
+        self.assertIs(pathlib.PurePosixPath,
+                      get_fs_path_class(fsspec.filesystem("s3")))
+        self.assertIs(pathlib.PurePosixPath,
+                      get_fs_path_class(fsspec.filesystem("memory")))

test/core/test_level.py

@@ -87,6 +87,13 @@ def test_write_read_levels_with_even_sizes(self):
         ]
         self._assert_io_ok(shape,
                            tile_shape,
+                           False,
+                           expected_num_levels,
+                           expected_shapes,
+                           expected_chunks)
+        self._assert_io_ok(shape,
+                           tile_shape,
+                           True,
                            expected_num_levels,
                            expected_shapes,
                            expected_chunks)
@@ -113,33 +120,43 @@ def test_write_read_levels_with_odd_sizes(self):
         ]
         self._assert_io_ok(shape,
                            tile_shape,
+                           False,
+                           expected_num_levels,
+                           expected_shapes,
+                           expected_chunks)
+        self._assert_io_ok(shape,
+                           tile_shape,
+                           True,
                            expected_num_levels,
                            expected_shapes,
                            expected_chunks)
 
     def _assert_io_ok(self,
                       shape,
                       tile_shape,
+                      link_input: bool,
                       expected_num_levels,
                       expected_shapes,
                       expected_chunks):
 
-        input_path = get_path("pyramid-input.nc")
+        input_path = get_path("pyramid-input.zarr")
         output_path = get_path("pyramid-output")
 
         rimraf(input_path)
         rimraf(output_path)
 
         try:
-            dataset = self.create_test_dataset(shape, chunks=None)
-            dataset.to_netcdf(input_path)
+            dataset = self.create_test_dataset(shape,
+                                               chunks=(1, *tile_shape))
+            dataset.to_zarr(input_path)
 
             t0 = time.perf_counter()
 
             levels = write_levels(output_path,
                                   dataset=dataset,
                                   spatial_tile_shape=tile_shape,
-                                  input_path=input_path)
+                                  input_path=input_path,
+                                  link_input=link_input)
 
             print(f"write time total: ", time.perf_counter() - t0)
 

xcube/core/level.py

@@ -244,6 +244,13 @@ def read_levels(
         if ext == ".link":
             with open(file_path, "r") as fp:
                 link_file_path = fp.read()
+            if not os.path.isabs(link_file_path):
+                parent_dir_path = os.path.abspath(
+                    os.path.dirname(dir_path) or '.'
+                )
+                link_file_path = os.path.join(
+                    parent_dir_path, link_file_path
+                )
             dataset = xr.open_zarr(link_file_path)
         else:
             dataset = xr.open_zarr(file_path)

xcube/core/mldataset.py

@@ -1,4 +1,24 @@
-import math
+# The MIT License (MIT)
+# Copyright (c) 2022 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 import os
 import threading
 import uuid
@@ -24,7 +44,6 @@
 from xcube.util.assertions import assert_instance
 from xcube.util.perf import measure_time
 from xcube.util.tilegrid import TileGrid
-from xcube.util.tilegrid import ImageTileGrid, TileGrid
 
 COMPUTE_DATASET = 'compute_dataset'
 
@@ -153,6 +172,14 @@ def tile_grid(self) -> TileGrid:
                 self._tile_grid = self._get_tile_grid_lazily()
         return self._tile_grid
 
+    @property
+    def lock(self) -> threading.RLock:
+        """
+        Get the reentrant lock used by this object to synchronize
+        lazy instantiation of properties.
+        """
+        return self._lock
+
     def get_dataset(self, index: int) -> xr.Dataset:
         """
         Get or compute the dataset for the level at given *index*.

xcube/core/store/fs/helpers.py

@@ -1,5 +1,5 @@
 # The MIT License (MIT)
-# Copyright (c) 2021 by the xcube development team and contributors
+# Copyright (c) 2022 by the xcube development team and contributors
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -19,13 +19,55 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import pathlib
+from typing import Type, Iterator
+
 import fsspec
 from fsspec.implementations.local import LocalFileSystem
 
 
 def is_local_fs(fs: fsspec.AbstractFileSystem) -> bool:
+    """
+    Check whether *fs* is a local filesystem.
+    """
     return fs.protocol == 'file' or isinstance(fs, LocalFileSystem)
 
 
-def normalize_path(path: str) -> str:
-    return path.replace('\\', '/') if '\\' in path else path
+def get_fs_path_class(fs: fsspec.AbstractFileSystem) \
+        -> Type[pathlib.PurePath]:
+    """
+    Get the appropriate ``pathlib.PurePath`` class for the filesystem *fs*.
+    """
+    if is_local_fs(fs):
+        # Will return PurePosixPath or a PureWindowsPath object
+        return pathlib.PurePath
+    # PurePosixPath for non-local filesystems such as S3,
+    # so we force use of forward slashes as separators
+    return pathlib.PurePosixPath
+
+
+def resolve_path(path: pathlib.PurePath) -> pathlib.PurePath:
+    """
+    Resolve "." and ".." occurrences in *path* without I/O and
+    return a new path.
+    """
+    reversed_parts = reversed(path.parts)
+    reversed_norm_parts = list(_resolve_path_impl(reversed_parts))
+    parts = reversed(reversed_norm_parts)
+    return type(path)(*parts)
+
+
+def _resolve_path_impl(reversed_parts: Iterator[str]):
+    skips = False
+    for part in reversed_parts:
+        if part == '.':
+            continue
+        elif part == '..':
+            skips += 1
+            continue
+        if skips == 0:
+            yield part
+        elif skips > 0:
+            skips -= 1
+    if skips != 0:
+        raise ValueError('cannot resolve path, misplaced ".."')