Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/3603.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Correct the target bytes number for auto-chunking when auto-sharding.
4 changes: 3 additions & 1 deletion src/zarr/core/chunk_grids.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ def _guess_chunks(
tuple[int, ...]

"""
if min_bytes >= max_bytes:
raise ValueError(f"Cannot have more min_bytes ({min_bytes}) than max_bytes ({max_bytes})")
if isinstance(shape, int):
shape = (shape,)

Expand Down Expand Up @@ -264,7 +266,7 @@ def _auto_partition(
else:
if chunk_shape == "auto":
# aim for a 1MiB chunk
_chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1024)
_chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1048576)
else:
_chunks_out = chunk_shape

Expand Down
18 changes: 18 additions & 0 deletions tests/test_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -1019,6 +1019,24 @@ def test_auto_partition_auto_shards(
assert auto_shards == expected_shards


def test_auto_partition_auto_shards_with_auto_chunks_should_be_close_to_1MiB() -> None:
"""
Test that automatically picking a shard size and a chunk size gives roughly 1MiB chunks.
"""
with pytest.warns(
ZarrUserWarning,
match="Automatic shard shape inference is experimental and may change without notice.",
):
with zarr.config.set({"array.target_shard_size_bytes": 10_000_000}):
_, chunk_shape = _auto_partition(
array_shape=(10_000_000,),
chunk_shape="auto",
shard_shape="auto",
item_size=1,
)
assert chunk_shape == (625000,)


def test_chunks_and_shards() -> None:
store = StorePath(MemoryStore())
shape = (100, 100)
Expand Down