Expose custom 'walkers' to CLI. (#551)

* Expose custom 'walkers' to CLI.

* Add skip_all and keep DEFAULT_WALKERS when overridded.

* Fix skip_all, and test it.

* whitespace

* Improved test, by @Wiebke

* Adjust skip_all to traverse subdirs.

tiled/_tests/test_directory_walker.py

@@ -5,7 +5,13 @@
 import tifffile
 
 from ..catalog import in_memory
-from ..catalog.register import identity, register, strip_suffixes
+from ..catalog.register import (
+    identity,
+    register,
+    skip_all,
+    strip_suffixes,
+    tiff_sequence,
+)
 from ..client import Context, from_context
 from ..examples.generate_files import data, df1, generate_files
 from ..server.app import build_app
@@ -112,3 +118,30 @@ def detect_mimetype(path, mimetype):
         )
         client = from_context(context)
         assert set(client) == {"a0", "a.0.asfwoeijviojefeiofw", "c.csv"}
+
+
+@pytest.mark.asyncio
+async def test_skip_all_in_combination(tmpdir):
+    "Using skip_all should override defaults, but not hinder other walkers"
+    df1.to_csv(Path(tmpdir, "a.csv"))
+    Path(tmpdir, "one").mkdir()
+    df1.to_csv(Path(tmpdir, "one", "a.csv"))
+
+    for i in range(2):
+        tifffile.imwrite(Path(tmpdir, "one", f"image{i:05}.tif"), data)
+
+    tree = in_memory()
+    # By default, both file and tiff sequence are registered.
+    with Context.from_app(build_app(tree)) as context:
+        await register(tree, tmpdir)
+        client = from_context(context)
+        assert "a" in client
+        assert "a" in client["one"]
+        assert "image" in client["one"]
+
+    # With skip_all, directories and tiff sequence are registered, but individual files are not
+    with Context.from_app(build_app(tree)) as context:
+        await register(tree, tmpdir, walkers=[tiff_sequence, skip_all])
+        client = from_context(context)
+        assert list(client) == ["one"]
+        assert "image" in client["one"]

tiled/catalog/register.py

@@ -152,8 +152,10 @@ async def register(
         filter=filter,
     )
     path = Path(path)
-    if walkers is None:
-        walkers = DEFAULT_WALKERS
+    parsed_walkers = []
+    for walker in walkers or []:
+        parsed_walkers.append(import_object(walker))
+    parsed_walkers.extend(DEFAULT_WALKERS)
     prefix_parts = [segment for segment in prefix.split("/") if segment]
     for segment in prefix_parts:
         child_catalog = await catalog.lookup_adapter([segment])
@@ -176,7 +178,7 @@ async def register(
         await _walk(
             catalog,
             Path(path),
-            walkers,
+            parsed_walkers,
             settings=settings,
         )
     else:
@@ -385,6 +387,23 @@ async def tiff_sequence(
     return unhandled_files, unhandled_directories
 
 
+async def skip_all(
+    catalog,
+    path,
+    files,
+    directories,
+    settings,
+):
+    """
+    Skip all files and directories without processing them.
+
+    This can be used to override the DEFAULT_WALKERS.
+    """
+    for item in files:
+        logger.info("    SKIP ALL: Nothing yet handled file '%s'", item)
+    return [], directories
+
+
 DEFAULT_WALKERS = [tiff_sequence, one_node_per_item]
 
 

tiled/commandline/_catalog.py

@@ -135,6 +135,16 @@ def register(
             "Specify here as 'mimetype=package.module:function'"
         ),
     ),
+    walkers: List[str] = typer.Option(
+        None,
+        "--walker",
+        help=(
+            "ADVANCED: Custom Tiled Walker for traversing directories and "
+            "grouping files. This is used in conjunction with Adapters that operate "
+            "on groups of files. "
+            "Specify here as 'package.module:function'"
+        ),
+    ),
 ):
     from ..catalog.utils import SCHEME_PATTERN
 
@@ -190,6 +200,7 @@ def register(
                 mimetype_detection_hook=mimetype_detection_hook,
                 mimetypes_by_file_ext=mimetypes_by_file_ext,
                 adapters_by_mimetype=adapters_by_mimetype,
+                walkers=walkers,
                 key_from_filename=key_from_filename,
             )
         )
@@ -202,6 +213,7 @@ def register(
                 mimetype_detection_hook=mimetype_detection_hook,
                 mimetypes_by_file_ext=mimetypes_by_file_ext,
                 adapters_by_mimetype=adapters_by_mimetype,
+                walkers=walkers,
                 key_from_filename=key_from_filename,
             )
         )

tiled/commandline/_serve.py

@@ -71,6 +71,16 @@ def serve_directory(
             "Specify here as 'mimetype=package.module:function'"
         ),
     ),
+    walkers: List[str] = typer.Option(
+        None,
+        "--walker",
+        help=(
+            "ADVANCED: Custom Tiled Walker for traversing directories and "
+            "grouping files. This is used in conjunction with Adapters that operate "
+            "on groups of files. "
+            "Specify here as 'package.module:function'"
+        ),
+    ),
     host: str = typer.Option(
         "127.0.0.1",
         help=(
@@ -189,6 +199,7 @@ async def walk_and_serve():
                     mimetype_detection_hook=mimetype_detection_hook,
                     mimetypes_by_file_ext=mimetypes_by_file_ext,
                     adapters_by_mimetype=adapters_by_mimetype,
+                    walkers=walkers,
                     key_from_filename=key_from_filename,
                 )
             )
@@ -211,6 +222,7 @@ async def walk_and_serve():
                 mimetype_detection_hook=mimetype_detection_hook,
                 mimetypes_by_file_ext=mimetypes_by_file_ext,
                 adapters_by_mimetype=adapters_by_mimetype,
+                walkers=walkers,
                 key_from_filename=key_from_filename,
             )
         )