Skip to content

Commit

Permalink
Datetime units (bluesky#782)
Browse files Browse the repository at this point in the history
* ENH: support units for numpy datetime types

* MNT: removed unnecessary imports

* ENH: add default value for units

* ENH: update BuiltinDtype in pydantic

* ENH: update BuiltinDtype in pydantic

* ENH: add default value for units

* TST: datetime dtypes in test_array

* MNT: Update changelog

* FIX: typo in comment

* MNT: fix changelog

* FIX: default value of units to empty string.

* FIX: use None as the sentinel for the units kwarg

* ENH: use np.datetime_data to extract units

* TST: Fix failing authorization test -- empty password

* MNT: format and lint
  • Loading branch information
genematx authored Sep 18, 2024
1 parent c44f3e2 commit 337ebab
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 21 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ Write the date in place of the "Unreleased" in the case a new version is release

# Changelog

## Unreleased

### Added

- Added support for explicit units in numpy datetime64 dtypes.

## v0.1.0b8 (2024-09-06)

### Fixed
Expand Down
8 changes: 3 additions & 5 deletions tiled/_tests/test_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,9 @@
"uint64": numpy.arange(10, dtype="uint64"),
"f": numpy.arange(10, dtype="f"),
"c": (numpy.arange(10) * 1j).astype("c"),
# "m": (
# numpy.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64') -
# numpy.datetime64('2008-01-01'),
# )
# "M": numpy.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64'),
"m": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]")
- numpy.datetime64("2008-01-01"),
"M": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]"),
"S": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="S3"),
"U": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="U3"),
}
Expand Down
8 changes: 2 additions & 6 deletions tiled/_tests/test_authentication.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@

import numpy
import pytest
from starlette.status import (
HTTP_400_BAD_REQUEST,
HTTP_401_UNAUTHORIZED,
HTTP_422_UNPROCESSABLE_ENTITY,
)
from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED

from ..adapters.array import ArrayAdapter
from ..adapters.mapping import MapAdapter
Expand Down Expand Up @@ -93,7 +89,7 @@ def test_password_auth(enter_password, config):
from_context(context, username="alice")

# Empty password should not work.
with fail_with_status_code(HTTP_422_UNPROCESSABLE_ENTITY):
with fail_with_status_code(HTTP_401_UNAUTHORIZED):
with enter_password(""):
from_context(context, username="alice")

Expand Down
12 changes: 11 additions & 1 deletion tiled/server/pydantic_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class BuiltinDtype(BaseModel):
endianness: Endianness
kind: Kind
itemsize: int
dt_units: Optional[str] = None

__endianness_map = {
">": "big",
Expand All @@ -39,10 +40,18 @@ class BuiltinDtype(BaseModel):

@classmethod
def from_numpy_dtype(cls, dtype) -> "BuiltinDtype":
# Extract datetime units from the dtype string representation,
# e.g. `'<M8[ns]'` has `dt_units = '[ns]'`. Count determines the number of base units in a step.
dt_units = None
if dtype.kind in ("m", "M"):
unit, count = numpy.datetime_data(dtype)
dt_units = f"[{count if count > 1 else ''}{unit}]"

return cls(
endianness=cls.__endianness_map[dtype.byteorder],
kind=Kind(dtype.kind),
itemsize=dtype.itemsize,
dt_units=dt_units,
)

def to_numpy_dtype(self):
Expand All @@ -60,14 +69,15 @@ def to_numpy_str(self):
# so the reported itemsize is 4x the char count. To get back to the string
# we need to divide by 4.
size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4
return f"{endianness}{self.kind.value}{size}"
return f"{endianness}{self.kind.value}{size}{self.dt_units or ''}"

@classmethod
def from_json(cls, structure):
return cls(
kind=Kind(structure["kind"]),
itemsize=structure["itemsize"],
endianness=Endianness(structure["endianness"]),
units=structure.get("dt_units"),
)


Expand Down
20 changes: 11 additions & 9 deletions tiled/structures/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ class BuiltinDtype:
endianness: Endianness
kind: Kind
itemsize: int
dt_units: Optional[str] = None

__endianness_map = {
">": "big",
Expand All @@ -88,15 +89,21 @@ class BuiltinDtype:

@classmethod
def from_numpy_dtype(cls, dtype) -> "BuiltinDtype":
# Extract datetime units from the dtype string representation,
# e.g. `'<M8[ns]'` has `dt_units = '[ns]'`. Count determines the number of base units in a step.
dt_units = None
if dtype.kind in ("m", "M"):
unit, count = numpy.datetime_data(dtype)
dt_units = f"[{count if count > 1 else ''}{unit}]"

return cls(
endianness=cls.__endianness_map[dtype.byteorder],
kind=Kind(dtype.kind),
itemsize=dtype.itemsize,
dt_units=dt_units,
)

def to_numpy_dtype(self) -> numpy.dtype:
import numpy

return numpy.dtype(self.to_numpy_str())

def to_numpy_str(self):
Expand All @@ -111,14 +118,15 @@ def to_numpy_str(self):
# so the reported itemsize is 4x the char count. To get back to the string
# we need to divide by 4.
size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4
return f"{endianness}{self.kind.value}{size}"
return f"{endianness}{self.kind.value}{size}{self.dt_units or ''}"

@classmethod
def from_json(cls, structure):
return cls(
kind=Kind(structure["kind"]),
itemsize=structure["itemsize"],
endianness=Endianness(structure["endianness"]),
dt_units=structure.get("dt_units"),
)


Expand All @@ -130,8 +138,6 @@ class Field:

@classmethod
def from_numpy_descr(cls, field):
import numpy

name, *rest = field
if name == "":
raise ValueError(
Expand Down Expand Up @@ -189,8 +195,6 @@ def from_numpy_dtype(cls, dtype):
)

def to_numpy_dtype(self):
import numpy

return numpy.dtype(self.to_numpy_descr())

def to_numpy_descr(self):
Expand Down Expand Up @@ -241,8 +245,6 @@ def from_array(cls, array, shape=None, chunks=None, dims=None) -> "ArrayStructur

if not hasattr(array, "__array__"):
# may be a list of something; convert to array
import numpy

array = numpy.asanyarray(array)

# Why would shape ever be different from array.shape, you ask?
Expand Down

0 comments on commit 337ebab

Please sign in to comment.