Skip to content

Commit

Permalink
Cleanup crc32c soft dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
jakirkham committed Nov 14, 2024
1 parent 29995e3 commit afce731
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 39 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,4 @@ repos:
hooks:
- id: mypy
args: [--config-file, pyproject.toml]
additional_dependencies: [numpy, pytest, zfpy, 'zarr==3.0.0b1']
additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr==3.0.0b1']
8 changes: 6 additions & 2 deletions numcodecs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,17 @@

register_codec(MsgPack)

from numcodecs.checksum32 import CRC32, CRC32C, Adler32, JenkinsLookup3
from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3

register_codec(CRC32)
register_codec(CRC32C)
register_codec(Adler32)
register_codec(JenkinsLookup3)

with suppress(ImportError):
from numcodecs.checksum32 import CRC32C

register_codec(CRC32C)

from numcodecs.json import JSON

register_codec(JSON)
Expand Down
46 changes: 23 additions & 23 deletions numcodecs/checksum32.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
import struct
import zlib
from collections.abc import Callable
from typing import TYPE_CHECKING, Literal
from contextlib import suppress
from types import ModuleType
from typing import TYPE_CHECKING, Literal, Optional

import numpy as np

from .abc import Codec
from .compat import ensure_contiguous_ndarray, ndarray_copy
from .jenkins import jenkins_lookup3

_crc32c: Optional[ModuleType] = None
with suppress(ImportError):
import crc32c as _crc32c # type: ignore[no-redef]

if TYPE_CHECKING:
from typing_extensions import Buffer

Expand Down Expand Up @@ -76,28 +82,6 @@ class CRC32(Checksum32):
location = 'start'


class CRC32C(Checksum32):
"""Codec add a crc32c checksum to the buffer.
Parameters
----------
location : 'start' or 'end'
Where to place the checksum in the buffer.
"""

codec_id = 'crc32c'

def checksum(self, buf):
try:
from crc32c import crc32c as crc32c_

return crc32c_(buf)
except ImportError: # pragma: no cover
raise ImportError("crc32c must be installed to use the CRC32C checksum codec.")

location = 'end'


class Adler32(Checksum32):
"""Codec add a adler32 checksum to the buffer.
Expand Down Expand Up @@ -168,3 +152,19 @@ def decode(self, buf, out=None):
out.view("uint8")[:] = b[:-4]
return out
return memoryview(b[:-4])


if _crc32c:

class CRC32C(Checksum32):
"""Codec add a crc32c checksum to the buffer.
Parameters
----------
location : 'start' or 'end'
Where to place the checksum in the buffer.
"""

codec_id = 'crc32c'
checksum = _crc32c.crc32c # type: ignore[union-attr]
location = 'end'
38 changes: 25 additions & 13 deletions numcodecs/tests/test_checksum32.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import itertools
from contextlib import suppress

import numpy as np
import pytest

try:
from numcodecs.checksum32 import CRC32, CRC32C, Adler32
except ImportError: # pragma: no cover
pytest.skip("numcodecs.checksum32 not available", allow_module_level=True)

from numcodecs.checksum32 import CRC32, Adler32
from numcodecs.tests.common import (
check_backwards_compatibility,
check_config,
Expand All @@ -17,6 +14,12 @@
check_repr,
)

has_crc32c = False
with suppress(ImportError):
from numcodecs.checksum32 import CRC32C

has_crc32c = True

# mix of dtypes: integer, float, bool, string
# mix of shapes: 1D, 2D, 3D
# mix of orders: C, F
Expand All @@ -39,11 +42,16 @@
codecs = [
CRC32(),
CRC32(location="end"),
CRC32C(location="start"),
CRC32C(),
Adler32(),
Adler32(location="end"),
]
if has_crc32c:
codecs.extend(
[
CRC32C(location="start"),
CRC32C(),
]
)


@pytest.mark.parametrize(("codec", "arr"), itertools.product(codecs, arrays))
Expand Down Expand Up @@ -88,25 +96,28 @@ def test_err_encode_list(codec):
def test_err_location():
with pytest.raises(ValueError):
CRC32(location="foo")
with pytest.raises(ValueError):
CRC32C(location="foo")
with pytest.raises(ValueError):
Adler32(location="foo")
if has_crc32c:
with pytest.raises(ValueError):
CRC32C(location="foo")


def test_repr():
check_repr("CRC32(location='start')")
check_repr("CRC32C(location='start')")
check_repr("Adler32(location='start')")
check_repr("CRC32(location='end')")
check_repr("CRC32C(location='end')")
check_repr("Adler32(location='start')")
check_repr("Adler32(location='end')")
if has_crc32c:
check_repr("CRC32C(location='start')")
check_repr("CRC32C(location='end')")


def test_backwards_compatibility():
check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()])
check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()])
check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])
if has_crc32c:
check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])


@pytest.mark.parametrize("codec", codecs)
Expand All @@ -127,6 +138,7 @@ def test_err_out_too_small(codec):
codec.decode(codec.encode(arr), out)


@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
def test_crc32c_checksum():
arr = np.arange(0, 64, dtype="uint8")
buf = CRC32C(location="end").encode(arr)
Expand Down

0 comments on commit afce731

Please sign in to comment.