From 002b80593285830662ae41d9089032143f103b14 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Wed, 16 Oct 2024 08:52:14 -0700 Subject: [PATCH 1/4] deps: slim zarr dependencies --- .pre-commit-config.yaml | 4 ---- pyproject.toml | 16 +++++++--------- src/zarr/codecs/crc32c_.py | 5 ++++- src/zarr/storage/remote.py | 4 ++-- tests/v3/test_buffer.py | 2 ++ tests/v3/test_codecs/test_codecs.py | 4 ++++ tests/v3/test_codecs/test_sharding.py | 2 ++ tests/v3/test_codecs/test_transpose.py | 2 ++ tests/v3/test_config.py | 3 +++ tests/v3/test_store/test_core.py | 2 +- tests/v3/test_store/test_remote.py | 6 +++--- 11 files changed, 30 insertions(+), 20 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dd5bd73df..4d9ba783b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,18 +28,14 @@ repos: files: src|tests additional_dependencies: # Package dependencies - - asciitree - crc32c - donfig - - fasteners - numcodecs - numpy - typing_extensions - universal-pathlib # Tests - pytest - # Zarr v2 - - types-redis - repo: https://github.com/scientific-python/cookie rev: 2024.08.19 hooks: diff --git a/pyproject.toml b/pyproject.toml index 3dc46a0f9c..1f1bb63c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,12 +23,8 @@ maintainers = [ requires-python = ">=3.11" # If you add a new dependency here, please also add it to .pre-commit-config.yml dependencies = [ - 'asciitree', 'numpy>=1.25', - 'fasteners', - 'numcodecs>=0.10.2', - 'fsspec>2024', - 'crc32c', + 'numcodecs>=0.12', 'typing_extensions', 'donfig', ] @@ -52,17 +48,19 @@ license = {text = "MIT License"} keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] +remote = [ + "fsspec", +] +sharding = [ + "crc32c", +] test = [ "coverage", "pytest", "pytest-cov", - "msgpack", - "lmdb", "s3fs", "pytest-asyncio", "moto[s3]", - "flask-cors", - "flask", "requests", "mypy", "hypothesis", diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index 3a6624ad25..16c88ab3d6 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -5,7 +5,6 @@ import numpy as np import typing_extensions -from crc32c import crc32c from zarr.abc.codec import BytesBytesCodec from zarr.core.common import JSON, parse_named_configuration @@ -35,6 +34,8 @@ async def _decode_single( chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: + from crc32c import crc32c + data = chunk_bytes.as_numpy_array() crc32_bytes = data[-4:] inner_bytes = data[:-4] @@ -53,6 +54,8 @@ async def _encode_single( chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: + from crc32c import crc32c + data = chunk_bytes.as_numpy_array() # Calculate the checksum and "cast" it to a numpy array checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32) diff --git a/src/zarr/storage/remote.py b/src/zarr/storage/remote.py index 0a0ec7f7cc..87a90fb911 100644 --- a/src/zarr/storage/remote.py +++ b/src/zarr/storage/remote.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING, Any, Self -import fsspec - from zarr.abc.store import ByteRangeRequest, Store from zarr.storage.common import _dereference_path @@ -130,6 +128,8 @@ def from_url( ------- RemoteStore """ + import fsspec + fs, path = fsspec.url_to_fs(url, **storage_options) return cls(fs=fs, path=path, mode=mode, allowed_exceptions=allowed_exceptions) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 60816d764e..3b87352581 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -97,6 +97,7 @@ async def test_async_array_gpu_prototype() -> None: @pytest.mark.asyncio async def test_codecs_use_of_prototype() -> None: + pytest.importorskip("crc32c") expect = np.zeros((10, 10), dtype="uint16", order="F") a = await AsyncArray.create( StorePath(StoreExpectingTestBuffer(mode="w")) / "test_codecs_use_of_prototype", @@ -132,6 +133,7 @@ async def test_codecs_use_of_prototype() -> None: @gpu_test @pytest.mark.asyncio async def test_codecs_use_of_gpu_prototype() -> None: + pytest.importorskip("crc32c") expect = cp.zeros((10, 10), dtype="uint16", order="F") a = await AsyncArray.create( StorePath(MemoryStore(mode="w")) / "test_codecs_use_of_gpu_prototype", diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index 7a5fb979a1..e320f57b77 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -73,6 +73,8 @@ async def test_order( runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: + if with_sharding: + pytest.importorskip("crc32c") data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) path = "order" spath = StorePath(store, path=path) @@ -129,6 +131,8 @@ def test_order_implicit( runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: + if with_sharding: + pytest.importorskip("crc32c") data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) path = "order_implicit" spath = StorePath(store, path) diff --git a/tests/v3/test_codecs/test_sharding.py b/tests/v3/test_codecs/test_sharding.py index c0dcfbf350..1aee25864f 100644 --- a/tests/v3/test_codecs/test_sharding.py +++ b/tests/v3/test_codecs/test_sharding.py @@ -20,6 +20,8 @@ from ..conftest import ArrayRequest from .test_codecs import _AsyncArrayProxy, order_from_dim +pytest.importorskip("crc32c") + @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/v3/test_codecs/test_transpose.py index 2b3914150e..314be02812 100644 --- a/tests/v3/test_codecs/test_transpose.py +++ b/tests/v3/test_codecs/test_transpose.py @@ -27,6 +27,8 @@ async def test_transpose( runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: + if with_sharding: + pytest.importorskip("crc32c") data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) spath = StorePath(store, path="transpose") codecs_: list[Codec] = ( diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py index 62907588c7..983867a71f 100644 --- a/tests/v3/test_config.py +++ b/tests/v3/test_config.py @@ -215,6 +215,9 @@ def test_config_buffer_implementation() -> None: arr[:] = np.arange(100) assert np.array_equal(arr[:], data) + +def test_config_sharding_implementation() -> None: + pytest.importorskip("crc32c") data2d = np.arange(1000).reshape(100, 10) arr_sharding = zeros( shape=(100, 10), diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py index b2a8292ea9..e72b72fa5f 100644 --- a/tests/v3/test_store/test_core.py +++ b/tests/v3/test_store/test_core.py @@ -39,7 +39,7 @@ async def test_make_store_path(tmpdir: str) -> None: async def test_make_store_path_fsspec(monkeypatch) -> None: - import fsspec.implementations.memory + fsspec = pytest.importorskip("fsspec") monkeypatch.setattr(fsspec.implementations.memory.MemoryFileSystem, "async_impl", True) store_path = await make_store_path("memory://") diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index c8e9a162b0..4e3c47cd93 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -4,10 +4,8 @@ import os from typing import TYPE_CHECKING -import fsspec import pytest from botocore.session import Session -from upath import UPath import zarr.api.asynchronous from zarr.core.buffer import Buffer, cpu, default_buffer_prototype @@ -21,6 +19,7 @@ import botocore.client +fsspec = pytest.importorskip("fsspec") s3fs = pytest.importorskip("s3fs") requests = pytest.importorskip("requests") moto_server = pytest.importorskip("moto.moto_server.threaded_moto_server") @@ -183,7 +182,8 @@ async def test_remote_store_from_uri( assert dict(group.attrs) == {"key": "value-3"} def test_from_upath(self) -> None: - path = UPath(f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False) + upath = pytest.importorskip("upath") + path = upath.UPath(f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False) result = RemoteStore.from_upath(path) assert result.fs.endpoint_url == endpoint_url From ad553c7bccf570e2dd7e9468219622becc9a6905 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 17 Oct 2024 09:58:51 -0700 Subject: [PATCH 2/4] fixup --- tests/v3/test_store/test_core.py | 4 ++-- tests/v3/test_store/test_zip.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py index d321f4b5e5..71ae5fb926 100644 --- a/tests/v3/test_store/test_core.py +++ b/tests/v3/test_store/test_core.py @@ -71,9 +71,9 @@ async def test_make_store_path_invalid() -> None: async def test_make_store_path_fsspec(monkeypatch) -> None: - fsspec = pytest.importorskip("fsspec") + memory = pytest.importorskip("fsspec.implementations.memory") - monkeypatch.setattr(fsspec.implementations.memory.MemoryFileSystem, "async_impl", True) + monkeypatch.setattr(memory.MemoryFileSystem, "async_impl", True) store_path = await make_store_path("memory://") assert isinstance(store_path.store, RemoteStore) diff --git a/tests/v3/test_store/test_zip.py b/tests/v3/test_store/test_zip.py index d05422ecde..eb129a1b2d 100644 --- a/tests/v3/test_store/test_zip.py +++ b/tests/v3/test_store/test_zip.py @@ -103,4 +103,4 @@ async def test_with_mode(self, store: ZipStore) -> None: @pytest.mark.parametrize("mode", ["a", "w"]) async def test_store_open_mode(self, store_kwargs: dict[str, Any], mode: str) -> None: - super().test_store_open_mode(store_kwargs, mode) + await super().test_store_open_mode(store_kwargs, mode) From c73c8b9e800a092ba532aed416df4bc188045e4c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 18 Oct 2024 06:10:09 -0700 Subject: [PATCH 3/4] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 686d525300..cfc90b0df3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ license = {text = "MIT License"} keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] -remote = [ +fsspec = [ "fsspec>=2023.10.0", ] sharding = [ From a164a73d2d71b1bb703ca918364c8385356de4ea Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 12 Nov 2024 22:19:35 -0800 Subject: [PATCH 4/4] revert crc32c changes --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cfc90b0df3..7eea48b8a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ 'numcodecs>=0.12', 'typing_extensions>=4.9', 'donfig>=0.8.1', + 'crc32c>=2.4', ] dynamic = [ "version", @@ -54,9 +55,6 @@ keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] fsspec = [ "fsspec>=2023.10.0", ] -sharding = [ - "crc32c>=2.4", -] test = [ "coverage", "pytest",