Skip to content

Commit

Permalink
Format (#26)
Browse files Browse the repository at this point in the history
  • Loading branch information
flying-sheep authored and LDeakin committed Nov 15, 2024
1 parent c3838e7 commit 3d88471
Show file tree
Hide file tree
Showing 13 changed files with 100 additions and 81 deletions.
3 changes: 0 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,9 @@ repos:
rev: v0.7.2
hooks:
- id: ruff
types_or: [python, pyi, jupyter]
args: ["--fix"]
- id: ruff-format
types_or: [python, pyi, jupyter]
# The following can be removed once PLR0917 is out of preview
- name: ruff preview rules
id: ruff
types_or: [python, pyi, jupyter]
args: ["--preview", "--select=PLR0917"]
18 changes: 11 additions & 7 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ test = [
"flask",
"requests",
"mypy",
"hypothesis"
"hypothesis",
]
dev = ["maturin"]

Expand All @@ -62,18 +62,20 @@ doctest_optionflags = [
"IGNORE_EXCEPTION_DETAIL",
]
addopts = [
"--durations=10", "-ra", "--strict-config", "--strict-markers",
"--durations=10",
"-ra",
"--strict-config",
"--strict-markers",
"--import-mode=importlib",
]
filterwarnings = [
"error:::zarr.*",
"ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
"ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
"ignore:Creating a zarr.buffer.gpu.*:UserWarning",
"ignore:Duplicate name:UserWarning", # from ZipFile
]
markers = [
"gpu: mark a test as requiring CuPy and GPU"
"ignore:Duplicate name:UserWarning", # from ZipFile
]
markers = ["gpu: mark a test as requiring CuPy and GPU"]

[tool.ruff]
src = ["src", "tests"]
Expand Down Expand Up @@ -106,4 +108,6 @@ ignore = [
"E262",
# allow I, O, l as variable names -> I is the identity matrix, i, j, k, l is reasonable indexing notation
"E741",
]
]
[tool.ruff.lint.isort]
known-first-party = ["zarrs_python"]
4 changes: 2 additions & 2 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ impl CodecPipelineImpl {
value_decoded: ArrayBytes,
codec_options: &CodecOptions,
) -> PyResult<()> {
if value_decoded.is_fill_value(&chunk_representation.fill_value()) {
store.erase(&key)
if value_decoded.is_fill_value(chunk_representation.fill_value()) {
store.erase(key)
} else {
let value_encoded = codec_chain
.encode(value_decoded, chunk_representation, codec_options)
Expand Down
29 changes: 11 additions & 18 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,43 +1,36 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import Literal
import pytest

from zarr.core.common import ChunkCoords, MemoryOrder

from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Literal

import numpy as np
import numpy.typing as npt
import pytest

from zarr import config
from zarr.abc.store import Store
from zarr.core.common import ChunkCoords
from zarr.storage import LocalStore, MemoryStore, ZipStore
from zarr.storage.remote import RemoteStore

import zarrs_python # noqa
import zarrs_python # noqa: F401

if TYPE_CHECKING:
from collections.abc import Generator
from typing import Any, Literal

from zarr.abc.store import Store
from zarr.core.common import ChunkCoords, MemoryOrder

from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat

@dataclass
class ArrayRequest:
shape: ChunkCoords
dtype: str
order: MemoryOrder


@pytest.fixture(autouse=True)
def run_around_each_test():
config.set({ "codec_pipeline.path": "zarrs_python.ZarrsCodecPipeline"})
yield
config.set({ "codec_pipeline.path": "zarrs_python.ZarrsCodecPipeline"})
def _setup_codec_pipeline():
config.set({"codec_pipeline.path": "zarrs_python.ZarrsCodecPipeline"})


async def parse_store(
store: Literal["local", "memory", "remote", "zip"], path: str
Expand All @@ -52,18 +45,18 @@ async def parse_store(
return await ZipStore.open(path + "/zarr.zip", mode="w")
raise AssertionError


@pytest.fixture(params=["local"])
async def store(request: pytest.FixtureRequest, tmpdir) -> Store:
param = request.param
return await parse_store(param, str(tmpdir))



@pytest.fixture
def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]:
array_request: ArrayRequest = request.param
return (
np.arange(np.prod(array_request.shape))
.reshape(array_request.shape, order=array_request.order)
.astype(array_request.dtype)
)
)
10 changes: 6 additions & 4 deletions tests/test_blosc.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@

import numpy as np
import pytest

from zarr import AsyncArray
from zarr.abc.store import Store
from zarr.codecs import BloscCodec, BytesCodec, ShardingCodec
from zarr.core.buffer import default_buffer_prototype
from zarr.storage.common import StorePath



@pytest.mark.parametrize("dtype", ["uint8", "uint16"])
async def test_blosc_evolve(store: Store, dtype: str) -> None:
typesize = np.dtype(dtype).itemsize
Expand Down Expand Up @@ -42,12 +40,16 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None:
chunk_shape=(16, 16),
dtype=dtype,
fill_value=0,
codecs=[ShardingCodec(chunk_shape=(16, 16), codecs=[BytesCodec(), BloscCodec()])],
codecs=[
ShardingCodec(chunk_shape=(16, 16), codecs=[BytesCodec(), BloscCodec()])
],
)
buf = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype())
assert buf is not None
zarr_json = json.loads(buf.to_bytes())
blosc_configuration_json = zarr_json["codecs"][0]["configuration"]["codecs"][1]["configuration"]
blosc_configuration_json = zarr_json["codecs"][0]["configuration"]["codecs"][1][
"configuration"
]
assert blosc_configuration_json["typesize"] == typesize
if typesize == 1:
assert blosc_configuration_json["shuffle"] == "bitshuffle"
Expand Down
54 changes: 34 additions & 20 deletions tests/test_codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@

import numpy as np
import pytest

from zarr import Array, AsyncArray, config
from zarr.codecs import (
BytesCodec,
GzipCodec,
ShardingCodec,
TransposeCodec,
)
Expand Down Expand Up @@ -59,13 +57,13 @@ def test_sharding_pickle() -> None:
pass



@pytest.mark.parametrize("input_order", ["F", "C"])
@pytest.mark.parametrize("store_order", ["F", "C"])
@pytest.mark.parametrize("runtime_write_order", ["C"])
@pytest.mark.parametrize("runtime_read_order", ["C"])
@pytest.mark.parametrize("with_sharding", [True, False])
async def test_order(
*,
store: Store,
input_order: MemoryOrder,
store_order: MemoryOrder,
Expand All @@ -80,11 +78,17 @@ async def test_order(
[
ShardingCodec(
chunk_shape=(16, 8),
codecs=[TransposeCodec(order=order_from_dim(store_order, data.ndim)), BytesCodec()],
codecs=[
TransposeCodec(order=order_from_dim(store_order, data.ndim)),
BytesCodec(),
],
)
]
if with_sharding
else [TransposeCodec(order=order_from_dim(store_order, data.ndim)), BytesCodec()]
else [
TransposeCodec(order=order_from_dim(store_order, data.ndim)),
BytesCodec(),
]
)

with config.set({"array.order": runtime_write_order}):
Expand Down Expand Up @@ -117,12 +121,12 @@ async def test_order(
assert read_data.flags["C_CONTIGUOUS"]



@pytest.mark.parametrize("input_order", ["F", "C"])
@pytest.mark.parametrize("runtime_write_order", ["C"])
@pytest.mark.parametrize("runtime_read_order", ["C"])
@pytest.mark.parametrize("with_sharding", [True, False])
def test_order_implicit(
*,
store: Store,
input_order: MemoryOrder,
runtime_write_order: MemoryOrder,
Expand All @@ -132,7 +136,9 @@ def test_order_implicit(
data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order)
path = "order_implicit"
spath = StorePath(store, path)
codecs_: list[Codec] | None = [ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None
codecs_: list[Codec] | None = (
[ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None
)

with config.set({"array.order": runtime_write_order}):
a = Array.create(
Expand All @@ -159,7 +165,6 @@ def test_order_implicit(
assert read_data.flags["C_CONTIGUOUS"]



def test_open(store: Store) -> None:
spath = StorePath(store)
a = Array.create(
Expand Down Expand Up @@ -205,7 +210,6 @@ def test_morton() -> None:
]



def test_write_partial_chunks(store: Store) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
spath = StorePath(store)
Expand All @@ -220,7 +224,6 @@ def test_write_partial_chunks(store: Store) -> None:
assert np.array_equal(a[0:16, 0:16], data)



async def test_delete_empty_chunks(store: Store) -> None:
data = np.ones((16, 16))
path = "delete_empty_chunks"
Expand All @@ -238,7 +241,6 @@ async def test_delete_empty_chunks(store: Store) -> None:
assert await store.get(f"{path}/c0/0", prototype=default_buffer_prototype()) is None



async def test_dimension_names(store: Store) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
path = "dimension_names"
Expand Down Expand Up @@ -267,12 +269,13 @@ async def test_dimension_names(store: Store) -> None:
)

assert (await AsyncArray.open(spath2)).metadata.dimension_names is None
zarr_json_buffer = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype())
zarr_json_buffer = await store.get(
f"{path2}/zarr.json", prototype=default_buffer_prototype()
)
assert zarr_json_buffer is not None
assert "dimension_names" not in json.loads(zarr_json_buffer.to_bytes())



def test_invalid_metadata(store: Store) -> None:
# LD: Disabled for `zarrs`. Including endianness for a single-byte data type is not invalid.
# spath2 = StorePath(store, "invalid_endian")
Expand Down Expand Up @@ -354,7 +357,6 @@ def test_invalid_metadata(store: Store) -> None:
# )



async def test_resize(store: Store) -> None:
data = np.zeros((16, 18), dtype="uint16")
path = "resize"
Expand All @@ -369,14 +371,26 @@ async def test_resize(store: Store) -> None:
)

await _AsyncArrayProxy(a)[:16, :18].set(data)
assert await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is not None
assert await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None
assert await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None
assert await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is not None
assert (
await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is not None
)
assert (
await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None
)
assert (
await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None
)
assert (
await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is not None
)

a = await a.resize((10, 12))
assert a.metadata.shape == (10, 12)
assert await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None
assert await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None
assert (
await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None
)
assert (
await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None
)
assert await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is None
assert await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is None
3 changes: 0 additions & 3 deletions tests/test_endian.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import numpy as np
import pytest

from zarr import AsyncArray
from zarr.abc.store import Store
from zarr.codecs import BytesCodec
Expand All @@ -11,7 +10,6 @@
from .test_codecs import _AsyncArrayProxy



@pytest.mark.parametrize("endian", ["big", "little"])
async def test_endian(store: Store, endian: Literal["big", "little"]) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
Expand All @@ -32,7 +30,6 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None:
assert np.array_equal(data, readback_data)



@pytest.mark.parametrize("dtype_input_endian", [">u2", "<u2"])
@pytest.mark.parametrize("dtype_store_endian", ["big", "little"])
async def test_endian_write(
Expand Down
3 changes: 0 additions & 3 deletions tests/test_gzip.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import numpy as np
import pytest

from zarr import Array
from zarr.abc.store import Store
from zarr.codecs import BytesCodec, GzipCodec
from zarr.storage.common import StorePath



def test_gzip(store: Store) -> None:
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))

Expand Down
3 changes: 2 additions & 1 deletion tests/test_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
import numpy as np
import pytest
import zarr
import zarrs_python # noqa: F401
from zarr.storage import LocalStore

import zarrs_python # noqa: F401


@pytest.fixture
def fill_value() -> int:
Expand Down
Loading

0 comments on commit 3d88471

Please sign in to comment.