diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4667b20de1..4d9ba783b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,6 @@ repos: files: src|tests additional_dependencies: # Package dependencies - - asciitree - crc32c - donfig - numcodecs @@ -37,8 +36,6 @@ repos: - universal-pathlib # Tests - pytest - # Zarr v2 - - types-redis - repo: https://github.com/scientific-python/cookie rev: 2024.08.19 hooks: diff --git a/pyproject.toml b/pyproject.toml index bee7c6abd5..7eea48b8a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,13 +26,11 @@ maintainers = [ requires-python = ">=3.11" # If you add a new dependency here, please also add it to .pre-commit-config.yml dependencies = [ - 'asciitree', 'numpy>=1.25', - 'numcodecs>=0.10.2', - 'fsspec>2024', - 'crc32c', - 'typing_extensions', - 'donfig', + 'numcodecs>=0.12', + 'typing_extensions>=4.9', + 'donfig>=0.8.1', + 'crc32c>=2.4', ] dynamic = [ "version", @@ -54,17 +52,16 @@ license = {text = "MIT License"} keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] +fsspec = [ + "fsspec>=2023.10.0", +] test = [ "coverage", "pytest", "pytest-cov", - "msgpack", - "lmdb", "s3fs", "pytest-asyncio", "moto[s3]", - "flask-cors", - "flask", "requests", "mypy", "hypothesis", diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index 3a6624ad25..16c88ab3d6 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -5,7 +5,6 @@ import numpy as np import typing_extensions -from crc32c import crc32c from zarr.abc.codec import BytesBytesCodec from zarr.core.common import JSON, parse_named_configuration @@ -35,6 +34,8 @@ async def _decode_single( chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: + from crc32c import crc32c + data = chunk_bytes.as_numpy_array() crc32_bytes = data[-4:] inner_bytes = data[:-4] @@ -53,6 +54,8 @@ async def _encode_single( chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: + from crc32c import crc32c + data = chunk_bytes.as_numpy_array() # Calculate the checksum and "cast" it to a numpy array checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32) diff --git a/src/zarr/storage/remote.py b/src/zarr/storage/remote.py index 0a0ec7f7cc..87a90fb911 100644 --- a/src/zarr/storage/remote.py +++ b/src/zarr/storage/remote.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING, Any, Self -import fsspec - from zarr.abc.store import ByteRangeRequest, Store from zarr.storage.common import _dereference_path @@ -130,6 +128,8 @@ def from_url( ------- RemoteStore """ + import fsspec + fs, path = fsspec.url_to_fs(url, **storage_options) return cls(fs=fs, path=path, mode=mode, allowed_exceptions=allowed_exceptions) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 60816d764e..3b87352581 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -97,6 +97,7 @@ async def test_async_array_gpu_prototype() -> None: @pytest.mark.asyncio async def test_codecs_use_of_prototype() -> None: + pytest.importorskip("crc32c") expect = np.zeros((10, 10), dtype="uint16", order="F") a = await AsyncArray.create( StorePath(StoreExpectingTestBuffer(mode="w")) / "test_codecs_use_of_prototype", @@ -132,6 +133,7 @@ async def test_codecs_use_of_prototype() -> None: @gpu_test @pytest.mark.asyncio async def test_codecs_use_of_gpu_prototype() -> None: + pytest.importorskip("crc32c") expect = cp.zeros((10, 10), dtype="uint16", order="F") a = await AsyncArray.create( StorePath(MemoryStore(mode="w")) / "test_codecs_use_of_gpu_prototype", diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index 7a5fb979a1..e320f57b77 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -73,6 +73,8 @@ async def test_order( runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: + if with_sharding: + pytest.importorskip("crc32c") data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) path = "order" spath = StorePath(store, path=path) @@ -129,6 +131,8 @@ def test_order_implicit( runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: + if with_sharding: + pytest.importorskip("crc32c") data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) path = "order_implicit" spath = StorePath(store, path) diff --git a/tests/v3/test_codecs/test_sharding.py b/tests/v3/test_codecs/test_sharding.py index c0dcfbf350..1aee25864f 100644 --- a/tests/v3/test_codecs/test_sharding.py +++ b/tests/v3/test_codecs/test_sharding.py @@ -20,6 +20,8 @@ from ..conftest import ArrayRequest from .test_codecs import _AsyncArrayProxy, order_from_dim +pytest.importorskip("crc32c") + @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/v3/test_codecs/test_transpose.py index 2b3914150e..314be02812 100644 --- a/tests/v3/test_codecs/test_transpose.py +++ b/tests/v3/test_codecs/test_transpose.py @@ -27,6 +27,8 @@ async def test_transpose( runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: + if with_sharding: + pytest.importorskip("crc32c") data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) spath = StorePath(store, path="transpose") codecs_: list[Codec] = ( diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py index 62907588c7..983867a71f 100644 --- a/tests/v3/test_config.py +++ b/tests/v3/test_config.py @@ -215,6 +215,9 @@ def test_config_buffer_implementation() -> None: arr[:] = np.arange(100) assert np.array_equal(arr[:], data) + +def test_config_sharding_implementation() -> None: + pytest.importorskip("crc32c") data2d = np.arange(1000).reshape(100, 10) arr_sharding = zeros( shape=(100, 10), diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py index 771dc3c43e..71ae5fb926 100644 --- a/tests/v3/test_store/test_core.py +++ b/tests/v3/test_store/test_core.py @@ -71,9 +71,9 @@ async def test_make_store_path_invalid() -> None: async def test_make_store_path_fsspec(monkeypatch) -> None: - import fsspec.implementations.memory + memory = pytest.importorskip("fsspec.implementations.memory") - monkeypatch.setattr(fsspec.implementations.memory.MemoryFileSystem, "async_impl", True) + monkeypatch.setattr(memory.MemoryFileSystem, "async_impl", True) store_path = await make_store_path("memory://") assert isinstance(store_path.store, RemoteStore) diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index c8e9a162b0..4e3c47cd93 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -4,10 +4,8 @@ import os from typing import TYPE_CHECKING -import fsspec import pytest from botocore.session import Session -from upath import UPath import zarr.api.asynchronous from zarr.core.buffer import Buffer, cpu, default_buffer_prototype @@ -21,6 +19,7 @@ import botocore.client +fsspec = pytest.importorskip("fsspec") s3fs = pytest.importorskip("s3fs") requests = pytest.importorskip("requests") moto_server = pytest.importorskip("moto.moto_server.threaded_moto_server") @@ -183,7 +182,8 @@ async def test_remote_store_from_uri( assert dict(group.attrs) == {"key": "value-3"} def test_from_upath(self) -> None: - path = UPath(f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False) + upath = pytest.importorskip("upath") + path = upath.UPath(f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False) result = RemoteStore.from_upath(path) assert result.fs.endpoint_url == endpoint_url diff --git a/tests/v3/test_store/test_zip.py b/tests/v3/test_store/test_zip.py index d05422ecde..eb129a1b2d 100644 --- a/tests/v3/test_store/test_zip.py +++ b/tests/v3/test_store/test_zip.py @@ -103,4 +103,4 @@ async def test_with_mode(self, store: ZipStore) -> None: @pytest.mark.parametrize("mode", ["a", "w"]) async def test_store_open_mode(self, store_kwargs: dict[str, Any], mode: str) -> None: - super().test_store_open_mode(store_kwargs, mode) + await super().test_store_open_mode(store_kwargs, mode)