diff --git a/changes/3603.bugfix.md b/changes/3603.bugfix.md new file mode 100644 index 0000000000..37e1da5cb1 --- /dev/null +++ b/changes/3603.bugfix.md @@ -0,0 +1 @@ +Correct the target bytes number for auto-chunking when auto-sharding. \ No newline at end of file diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index 7ebd68b5b4..2c7945fa64 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -62,6 +62,8 @@ def _guess_chunks( tuple[int, ...] """ + if min_bytes >= max_bytes: + raise ValueError(f"Cannot have more min_bytes ({min_bytes}) than max_bytes ({max_bytes})") if isinstance(shape, int): shape = (shape,) @@ -264,7 +266,7 @@ def _auto_partition( else: if chunk_shape == "auto": # aim for a 1MiB chunk - _chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1024) + _chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1048576) else: _chunks_out = chunk_shape diff --git a/tests/test_array.py b/tests/test_array.py index 61828be0aa..67be294827 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1019,6 +1019,24 @@ def test_auto_partition_auto_shards( assert auto_shards == expected_shards +def test_auto_partition_auto_shards_with_auto_chunks_should_be_close_to_1MiB() -> None: + """ + Test that automatically picking a shard size and a chunk size gives roughly 1MiB chunks. + """ + with pytest.warns( + ZarrUserWarning, + match="Automatic shard shape inference is experimental and may change without notice.", + ): + with zarr.config.set({"array.target_shard_size_bytes": 10_000_000}): + _, chunk_shape = _auto_partition( + array_shape=(10_000_000,), + chunk_shape="auto", + shard_shape="auto", + item_size=1, + ) + assert chunk_shape == (625000,) + + def test_chunks_and_shards() -> None: store = StorePath(MemoryStore()) shape = (100, 100)