From 8e230899de12f2aafc18f40f058120658b715b9e Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:30:09 +0200 Subject: [PATCH 01/47] Update _typing.py --- xarray/namedarray/_typing.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index b715973814f..4e091482f76 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -43,9 +43,30 @@ class Default(Enum): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) + +class _DType2(Protocol): + def __eq__(self, other: DType2, /) -> bool: + """ + Computes the truth value of ``self == other`` in order to test for data type object equality. + + Parameters + ---------- + self: dtype + data type instance. May be any supported data type. + other: dtype + other data type instance. May be any supported data type. + + Returns + ------- + out: bool + a boolean indicating whether the data type objects are equal. + """ + ... + + _dtype = np.dtype -_DType = TypeVar("_DType", bound=np.dtype[Any]) -_DType_co = TypeVar("_DType_co", covariant=True, bound=np.dtype[Any]) +_DType = TypeVar("_DType", bound=_dtype[Any]) +_DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[Any]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` _ScalarType = TypeVar("_ScalarType", bound=np.generic) @@ -60,9 +81,9 @@ def dtype(self) -> _DType_co: ... _DTypeLike = Union[ - np.dtype[_ScalarType], + _dtype[_ScalarType], type[_ScalarType], - _SupportsDType[np.dtype[_ScalarType]], + _SupportsDType[_dtype[_ScalarType]], ] # For unknown shapes Dask uses np.nan, array_api uses None: @@ -216,7 +237,7 @@ def __array_namespace__(self) -> ModuleType: ... ] # Corresponds to np.typing.NDArray: -DuckArray = _arrayfunction[Any, np.dtype[_ScalarType_co]] +DuckArray = _arrayfunction[Any, _dtype[_ScalarType_co]] @runtime_checkable From 7f50c5ad6b26e546c8f23422fcd63e0472988516 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:31:59 +0200 Subject: [PATCH 02/47] Update _typing.py --- xarray/namedarray/_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 4e091482f76..9660023a454 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -45,7 +45,7 @@ class Default(Enum): class _DType2(Protocol): - def __eq__(self, other: DType2, /) -> bool: + def __eq__(self, other: _DType2, /) -> bool: """ Computes the truth value of ``self == other`` in order to test for data type object equality. From 66368541feb108962da347ea8ed50addc554fb57 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:38:44 +0200 Subject: [PATCH 03/47] Update core.py --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 135dabc0656..75fc74247a6 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -1129,7 +1129,7 @@ def expand_dims( return expand_dims(self, dim=dim) -_NamedArray = NamedArray[Any, np.dtype[_ScalarType_co]] +_NamedArray = NamedArray[Any, _dtype[_ScalarType_co]] def _raise_if_any_duplicate_dimensions( From bff34377efd5a5d135b5f66252cad33c285ca739 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:41:10 +0200 Subject: [PATCH 04/47] Use strict dtype, lets' how much breaks --- xarray/namedarray/_typing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 9660023a454..6ce57f4de54 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -44,7 +44,7 @@ class Default(Enum): _T_co = TypeVar("_T_co", covariant=True) -class _DType2(Protocol): +class _DType2(Protocol[Any]): def __eq__(self, other: _DType2, /) -> bool: """ Computes the truth value of ``self == other`` in order to test for data type object equality. @@ -64,7 +64,8 @@ def __eq__(self, other: _DType2, /) -> bool: ... -_dtype = np.dtype +# _dtype = np.dtype +_dtype = _DType2 _DType = TypeVar("_DType", bound=_dtype[Any]) _DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[Any]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` From 7028a53092ff63f0a7e4176f08543a1249f51fe0 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:44:34 +0200 Subject: [PATCH 05/47] Update _typing.py --- xarray/namedarray/_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 6ce57f4de54..d678d63e6cc 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -44,8 +44,8 @@ class Default(Enum): _T_co = TypeVar("_T_co", covariant=True) -class _DType2(Protocol[Any]): - def __eq__(self, other: _DType2, /) -> bool: +class _DType2(Protocol[_T]): + def __eq__(self, other: _DType2[Any], /) -> bool: """ Computes the truth value of ``self == other`` in order to test for data type object equality. From 8a4f62c3d452c2d5c08223b8258748d2b31fdad6 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:54:33 +0200 Subject: [PATCH 06/47] Update _typing.py --- xarray/namedarray/_typing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index d678d63e6cc..23dc8a2f7fc 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -44,7 +44,7 @@ class Default(Enum): _T_co = TypeVar("_T_co", covariant=True) -class _DType2(Protocol[_T]): +class _DType2(Protocol[_T_co]): def __eq__(self, other: _DType2[Any], /) -> bool: """ Computes the truth value of ``self == other`` in order to test for data type object equality. @@ -175,14 +175,14 @@ def __getitem__( ) -> _arrayfunction[Any, _DType_co] | Any: ... @overload - def __array__(self, dtype: None = ..., /) -> np.ndarray[Any, _DType_co]: ... + def __array__(self, dtype: None = ..., /) -> np.ndarray[Any, Any]: ... @overload - def __array__(self, dtype: _DType, /) -> np.ndarray[Any, _DType]: ... + def __array__(self, dtype: Any, /) -> np.ndarray[Any, Any]: ... def __array__( - self, dtype: _DType | None = ..., / - ) -> np.ndarray[Any, _DType] | np.ndarray[Any, _DType_co]: ... + self, dtype: Any | None = ..., / + ) -> np.ndarray[Any, Any] | np.ndarray[Any, Any]: ... # TODO: Should return the same subclass but with a new dtype generic. # https://github.com/python/typing/issues/548 From adda1ba042f0b22b9b19a56711261a6efdfeeacc Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 19:55:39 +0200 Subject: [PATCH 07/47] Update _typing.py --- xarray/namedarray/_typing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 23dc8a2f7fc..2afeaa70342 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -301,7 +301,7 @@ class _sparsearray( Corresponds to np.ndarray. """ - def todense(self) -> np.ndarray[Any, _DType_co]: ... + def todense(self) -> np.ndarray[Any, Any]: ... @runtime_checkable @@ -314,7 +314,7 @@ class _sparsearrayfunction( Corresponds to np.ndarray. """ - def todense(self) -> np.ndarray[Any, _DType_co]: ... + def todense(self) -> np.ndarray[Any, Any]: ... @runtime_checkable @@ -327,7 +327,7 @@ class _sparsearrayapi( Corresponds to np.ndarray. """ - def todense(self) -> np.ndarray[Any, _DType_co]: ... + def todense(self) -> np.ndarray[Any, Any]: ... # NamedArray can most likely use both __array_function__ and __array_namespace__: From cd095f26e577e36ccdb044e7e4f4f7bd29486c52 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 22:58:01 +0200 Subject: [PATCH 08/47] Update _typing.py --- xarray/namedarray/_typing.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 2afeaa70342..92d1c1dd8c0 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -43,9 +43,11 @@ class Default(Enum): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) +_generic = Any + class _DType2(Protocol[_T_co]): - def __eq__(self, other: _DType2[Any], /) -> bool: + def __eq__(self, other: _DType2[_generic], /) -> bool: """ Computes the truth value of ``self == other`` in order to test for data type object equality. @@ -66,12 +68,12 @@ def __eq__(self, other: _DType2[Any], /) -> bool: # _dtype = np.dtype _dtype = _DType2 -_DType = TypeVar("_DType", bound=_dtype[Any]) -_DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[Any]) +_DType = TypeVar("_DType", bound=_dtype[_generic]) +_DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[_generic]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` -_ScalarType = TypeVar("_ScalarType", bound=np.generic) -_ScalarType_co = TypeVar("_ScalarType_co", bound=np.generic, covariant=True) +_ScalarType = TypeVar("_ScalarType", bound=_generic) +_ScalarType_co = TypeVar("_ScalarType_co", bound=_generic, covariant=True) # A protocol for anything with the dtype attribute @@ -174,15 +176,9 @@ def __getitem__( /, ) -> _arrayfunction[Any, _DType_co] | Any: ... - @overload - def __array__(self, dtype: None = ..., /) -> np.ndarray[Any, Any]: ... - - @overload - def __array__(self, dtype: Any, /) -> np.ndarray[Any, Any]: ... - def __array__( - self, dtype: Any | None = ..., / - ) -> np.ndarray[Any, Any] | np.ndarray[Any, Any]: ... + self, dtype: _dtype | None = ..., / + ) -> np.ndarray[Any, np.dtype[np.generic]]: ... # TODO: Should return the same subclass but with a new dtype generic. # https://github.com/python/typing/issues/548 @@ -301,7 +297,7 @@ class _sparsearray( Corresponds to np.ndarray. """ - def todense(self) -> np.ndarray[Any, Any]: ... + def todense(self) -> np.ndarray[Any, np.dtype[np.generic]]: ... @runtime_checkable @@ -314,7 +310,7 @@ class _sparsearrayfunction( Corresponds to np.ndarray. """ - def todense(self) -> np.ndarray[Any, Any]: ... + def todense(self) -> np.ndarray[Any, np.dtype[np.generic]]: ... @runtime_checkable @@ -327,7 +323,7 @@ class _sparsearrayapi( Corresponds to np.ndarray. """ - def todense(self) -> np.ndarray[Any, Any]: ... + def todense(self) -> np.ndarray[Any, np.dtype[np.generic]]: ... # NamedArray can most likely use both __array_function__ and __array_namespace__: From 519702c87443111af09307292b9c793f55c96265 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:16:52 +0200 Subject: [PATCH 09/47] Update _typing.py --- xarray/namedarray/_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 92d1c1dd8c0..3cab1b2067a 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -177,7 +177,7 @@ def __getitem__( ) -> _arrayfunction[Any, _DType_co] | Any: ... def __array__( - self, dtype: _dtype | None = ..., / + self, dtype: _dtype[_generic] | None = ..., / ) -> np.ndarray[Any, np.dtype[np.generic]]: ... # TODO: Should return the same subclass but with a new dtype generic. From 6c64a4af08043031858be5feec79efda049ff0cb Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:40:47 +0200 Subject: [PATCH 10/47] Update pycompat.py --- xarray/namedarray/pycompat.py | 39 ++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 3ce33d4d8ea..1df82b5940a 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -2,7 +2,7 @@ from importlib import import_module from types import ModuleType -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, overload import numpy as np from packaging.version import Version @@ -15,7 +15,16 @@ if TYPE_CHECKING: ModType = Literal["dask", "pint", "cupy", "sparse", "cubed", "numbagg"] DuckArrayTypes = tuple[type[Any], ...] # TODO: improve this? maybe Generic - from xarray.namedarray._typing import _DType, _ShapeType, duckarray + from xarray.core.indexing import ExplicitlyIndexed + + from xarray.namedarray._typing import ( + _DType, + _ShapeType, + duckarray, + chunkedduckarray, + _dtype, + _generic, + ) class DuckArrayModule: @@ -121,7 +130,31 @@ def to_numpy( return data -def to_duck_array(data: Any, **kwargs: dict[str, Any]) -> duckarray[_ShapeType, _DType]: +@overload +def to_duck_array( + data: ExplicitlyIndexed, **kwargs: dict[str, Any] +) -> duckarray[Any, _dtype[_generic]]: ... +@overload +def to_duck_array( + data: chunkedduckarray[_ShapeType, _DType], **kwargs: dict[str, Any] +) -> chunkedduckarray[_ShapeType, _DType]: ... +@overload +def to_duck_array( + data: duckarray[_ShapeType, _DType], **kwargs: dict[str, Any] +) -> duckarray[_ShapeType, _DType]: ... +@overload +def to_duck_array( + data: np.typing.ArrayLike, **kwargs: dict[str, Any] +) -> duckarray[Any, _dtype[_generic]]: ... +def to_duck_array( + data: ( + ExplicitlyIndexed + | chunkedduckarray[_ShapeType, _DType] + | duckarray[_ShapeType, _DType] + | np.typing.ArrayLike + ), + **kwargs: dict[str, Any], +) -> duckarray[_ShapeType, _DType] | duckarray[Any, _dtype[_generic]]: from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.parallelcompat import get_chunked_array_type From 65f1c78a07bc63f05a8b31756ad4461caaad0210 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:42:22 +0000 Subject: [PATCH 11/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/pycompat.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 1df82b5940a..7d32c8380b6 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -16,14 +16,13 @@ ModType = Literal["dask", "pint", "cupy", "sparse", "cubed", "numbagg"] DuckArrayTypes = tuple[type[Any], ...] # TODO: improve this? maybe Generic from xarray.core.indexing import ExplicitlyIndexed - from xarray.namedarray._typing import ( _DType, - _ShapeType, - duckarray, - chunkedduckarray, _dtype, _generic, + _ShapeType, + chunkedduckarray, + duckarray, ) From 64ef10b7214973a9b5c21b0d11e1ae330ceda4d0 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 14 Apr 2024 21:25:17 +0200 Subject: [PATCH 12/47] Type hint run on parallelcompat --- xarray/namedarray/_typing.py | 20 +++++- xarray/namedarray/daskmanager.py | 103 +++++++++++++++------------- xarray/namedarray/parallelcompat.py | 95 ++++++++++++------------- 3 files changed, 117 insertions(+), 101 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 3cab1b2067a..2fae69710a9 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -100,8 +100,14 @@ def dtype(self) -> _DType_co: ... _Axes = tuple[_Axis, ...] _AxisLike = Union[_Axis, _Axes] -_Chunks = tuple[_Shape, ...] -_NormalizedChunks = tuple[tuple[int, ...], ...] +_Chunk = tuple[int, ...] +_Chunks = tuple[_Chunk, ...] +_NormalizedChunks = tuple[tuple[int, ...], ...] # TODO: Same as Chunks. +_ChunksLike = Union[ + int, Literal["auto"], None, _chunk, _Chunks +] # TODO: Literal["auto"] +_ChunksType = TypeVar("_ChunksType", bound=_Chunks) + # FYI in some cases we don't allow `None`, which this doesn't take account of. T_ChunkDim: TypeAlias = Union[int, Literal["auto"], None, tuple[int, ...]] # We allow the tuple form of this (though arguably we could transition to named dims only) @@ -264,6 +270,11 @@ class _chunkedarrayfunction( @property def chunks(self) -> _Chunks: ... + def rechunk( + self, + chunks: _ChunksLike, + ) -> _chunkedarrayfunction[_ShapeType_co, _DType_co]: ... + @runtime_checkable class _chunkedarrayapi( @@ -278,6 +289,11 @@ class _chunkedarrayapi( @property def chunks(self) -> _Chunks: ... + def rechunk( + self, + chunks: _ChunksLike, + ) -> _chunkedarrayapi[_ShapeType_co, _DType_co]: ... + # NamedArray can most likely use both __array_function__ and __array_namespace__: _chunkedarrayfunction_or_api = (_chunkedarrayfunction, _chunkedarrayapi) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index 14744d2de6b..6af8cd82252 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -1,57 +1,67 @@ from __future__ import annotations from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any, Callable, TypeVar, overload import numpy as np from packaging.version import Version from xarray.core.indexing import ImplicitToExplicitIndexingAdapter -from xarray.namedarray.parallelcompat import ChunkManagerEntrypoint, T_ChunkedArray +from xarray.namedarray.parallelcompat import ChunkManagerEntrypoint from xarray.namedarray.utils import is_duck_dask_array, module_available if TYPE_CHECKING: from xarray.namedarray._typing import ( T_Chunks, + _Chunks, + _ChunksLike, + _ChunksType, + _dtype, + _DType, _DType_co, _NormalizedChunks, + _Shape, + _ShapeType, + chunkedduckarray, duckarray, ) try: - from dask.array import Array as DaskArray + from dask.array.core import Array as DaskArray except ImportError: DaskArray = np.ndarray[Any, Any] # type: ignore[assignment, misc] dask_available = module_available("dask") +_T_DaskArray = TypeVar("_T_DaskArray", bound="DaskArray") -class DaskManager(ChunkManagerEntrypoint["DaskArray"]): # type: ignore[type-var] - array_cls: type[DaskArray] + +class DaskManager(ChunkManagerEntrypoint): # type: ignore[type-var] + # array_cls: type[DaskArray] available: bool = dask_available def __init__(self) -> None: # TODO can we replace this with a class attribute instead? - from dask.array import Array + from dask.array.core import Array self.array_cls = Array def is_chunked_array(self, data: duckarray[Any, Any]) -> bool: return is_duck_dask_array(data) - def chunks(self, data: Any) -> _NormalizedChunks: + def chunks(self, data: chunkedduckarray[Any, Any]) -> _Chunks: return data.chunks # type: ignore[no-any-return] def normalize_chunks( self, - chunks: T_Chunks | _NormalizedChunks, - shape: tuple[int, ...] | None = None, + chunks: _ChunksLike, + shape: _Shape | None = None, limit: int | None = None, - dtype: _DType_co | None = None, - previous_chunks: _NormalizedChunks | None = None, - ) -> Any: + dtype: _dtype | None = None, + previous_chunks: _Chunks | None = None, + ) -> _Chunks: """Called by open_dataset""" from dask.array.core import normalize_chunks @@ -64,24 +74,24 @@ def normalize_chunks( ) # type: ignore[no-untyped-call] def from_array( - self, data: Any, chunks: T_Chunks | _NormalizedChunks, **kwargs: Any - ) -> DaskArray | Any: - import dask.array as da + self, data: duckarray[Any, _DType], chunks: _ChunksLike, **kwargs: Any + ) -> chunkedduckarray[Any, _DType]: + from dask.array.core import from_array if isinstance(data, ImplicitToExplicitIndexingAdapter): # lazily loaded backend array classes should use NumPy array operations. kwargs["meta"] = np.ndarray - return da.from_array( + return from_array( data, chunks, **kwargs, ) # type: ignore[no-untyped-call] def compute( - self, *data: Any, **kwargs: Any - ) -> tuple[np.ndarray[Any, _DType_co], ...]: - from dask.array import compute + self, *data: chunkedduckarray[Any, _DType] | Any, **kwargs: Any + ) -> tuple[np.ndarray[Any, np.dtype[np.generic]], ...]: + from dask.base import compute return compute(*data, **kwargs) # type: ignore[no-untyped-call, no-any-return] @@ -91,17 +101,17 @@ def array_api(self) -> Any: return da - def reduction( # type: ignore[override] + def reduction( self, - arr: T_ChunkedArray, + arr: chunkedduckarray[Any, _DType], func: Callable[..., Any], combine_func: Callable[..., Any] | None = None, aggregate_func: Callable[..., Any] | None = None, axis: int | Sequence[int] | None = None, - dtype: _DType_co | None = None, + dtype: _DType | None = None, keepdims: bool = False, - ) -> DaskArray | Any: - from dask.array import reduction + ) -> chunkedduckarray[Any, _DType]: + from dask.array.reductions import reduction return reduction( arr, @@ -113,16 +123,16 @@ def reduction( # type: ignore[override] keepdims=keepdims, ) # type: ignore[no-untyped-call] - def scan( # type: ignore[override] + def scan( self, func: Callable[..., Any], binop: Callable[..., Any], ident: float, - arr: T_ChunkedArray, + arr: chunkedduckarray[Any, _DType], axis: int | None = None, - dtype: _DType_co | None = None, + dtype: _DType | None = None, **kwargs: Any, - ) -> DaskArray | Any: + ) -> chunkedduckarray[Any, _DType]: from dask.array.reductions import cumreduction return cumreduction( @@ -141,15 +151,15 @@ def apply_gufunc( signature: str, *args: Any, axes: Sequence[tuple[int, ...]] | None = None, - axis: int | None = None, keepdims: bool = False, - output_dtypes: Sequence[_DType_co] | None = None, - output_sizes: dict[str, int] | None = None, + output_dtypes: Sequence[_DType] | None = None, vectorize: bool | None = None, + axis: int | None = None, + output_sizes: dict[str, int] | None = None, allow_rechunk: bool = False, - meta: tuple[np.ndarray[Any, _DType_co], ...] | None = None, + meta: tuple[np.ndarray[Any, np.dtype[np.generic]], ...] | None = None, **kwargs: Any, - ) -> Any: + ) -> chunkedduckarray[Any, _DType] | tuple[chunkedduckarray[Any, _DType], ...]: from dask.array.gufunc import apply_gufunc return apply_gufunc( @@ -171,14 +181,14 @@ def map_blocks( self, func: Callable[..., Any], *args: Any, - dtype: _DType_co | None = None, - chunks: tuple[int, ...] | None = None, + dtype: _DType | None = None, + chunks: _Chunks | None = None, drop_axis: int | Sequence[int] | None = None, new_axis: int | Sequence[int] | None = None, **kwargs: Any, - ) -> Any: + ) -> chunkedduckarray[Any, _DType]: import dask - from dask.array import map_blocks + from dask.array.core import map_blocks if drop_axis is None and Version(dask.__version__) < Version("2022.9.1"): # See https://github.com/pydata/xarray/pull/7019#discussion_r1196729489 @@ -200,19 +210,18 @@ def blockwise( self, func: Callable[..., Any], out_ind: Iterable[Any], - *args: Any, - # can't type this as mypy assumes args are all same type, but dask blockwise args alternate types - name: str | None = None, - token: Any | None = None, - dtype: _DType_co | None = None, + *args: Any, # can't type this as mypy assumes args are all same type, but dask blockwise args alternate types adjust_chunks: dict[Any, Callable[..., Any]] | None = None, new_axes: dict[Any, int] | None = None, align_arrays: bool = True, + name: str | None = None, + token: Any | None = None, + dtype: _DType | None = None, concatenate: bool | None = None, - meta: tuple[np.ndarray[Any, _DType_co], ...] | None = None, + meta: tuple[np.ndarray[Any, np.dtype[np.generic]], ...] | None = None, **kwargs: Any, - ) -> DaskArray | Any: - from dask.array import blockwise + ) -> chunkedduckarray[Any, _DType]: + from dask.array.blockwise import blockwise return blockwise( func, @@ -233,7 +242,7 @@ def unify_chunks( self, *args: Any, # can't type this as mypy assumes args are all same type, but dask unify_chunks args alternate types **kwargs: Any, - ) -> tuple[dict[str, _NormalizedChunks], list[DaskArray]]: + ) -> tuple[dict[str, _Chunks], list[chunkedduckarray[Any, Any]]]: from dask.array.core import unify_chunks return unify_chunks(*args, **kwargs) # type: ignore[no-any-return, no-untyped-call] @@ -244,7 +253,7 @@ def store( targets: Any, **kwargs: Any, ) -> Any: - from dask.array import store + from dask.array.core import store return store( sources=sources, diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index dd555fe200a..0abea576180 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -11,7 +11,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable, Sequence from importlib.metadata import EntryPoint, entry_points -from typing import TYPE_CHECKING, Any, Callable, Generic, Protocol, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Generic, Protocol, TypeVar, overload import numpy as np @@ -21,33 +21,22 @@ if TYPE_CHECKING: from xarray.namedarray._typing import ( _Chunks, + _ChunksLike, + _ChunksType, _DType, _DType_co, _NormalizedChunks, + _Shape, _ShapeType, + _ShapeType_co, duckarray, + _chunkedarrayfunction_or_api, + chunkedduckarray, ) -class ChunkedArrayMixinProtocol(Protocol): - def rechunk(self, chunks: Any, **kwargs: Any) -> Any: ... - - @property - def dtype(self) -> np.dtype[Any]: ... - - @property - def chunks(self) -> _NormalizedChunks: ... - - def compute( - self, *data: Any, **kwargs: Any - ) -> tuple[np.ndarray[Any, _DType_co], ...]: ... - - -T_ChunkedArray = TypeVar("T_ChunkedArray", bound=ChunkedArrayMixinProtocol) - - @functools.lru_cache(maxsize=1) -def list_chunkmanagers() -> dict[str, ChunkManagerEntrypoint[Any]]: +def list_chunkmanagers() -> dict[str, ChunkManagerEntrypoint]: """ Return a dictionary of available chunk managers and their ChunkManagerEntrypoint subclass objects. @@ -71,7 +60,7 @@ def list_chunkmanagers() -> dict[str, ChunkManagerEntrypoint[Any]]: def load_chunkmanagers( entrypoints: Sequence[EntryPoint], -) -> dict[str, ChunkManagerEntrypoint[Any]]: +) -> dict[str, ChunkManagerEntrypoint]: """Load entrypoints and instantiate chunkmanagers only once.""" loaded_entrypoints = {} @@ -93,8 +82,8 @@ def load_chunkmanagers( def guess_chunkmanager( - manager: str | ChunkManagerEntrypoint[Any] | None, -) -> ChunkManagerEntrypoint[Any]: + manager: str | ChunkManagerEntrypoint | None, +) -> ChunkManagerEntrypoint: """ Get namespace of chunk-handling methods, guessing from what's available. @@ -128,7 +117,7 @@ def guess_chunkmanager( ) -def get_chunked_array_type(*args: Any) -> ChunkManagerEntrypoint[Any]: +def get_chunked_array_type(*args: Any) -> ChunkManagerEntrypoint: """ Detects which parallel backend should be used for given set of arrays. @@ -171,7 +160,7 @@ def get_chunked_array_type(*args: Any) -> ChunkManagerEntrypoint[Any]: return selected[0] -class ChunkManagerEntrypoint(ABC, Generic[T_ChunkedArray]): +class ChunkManagerEntrypoint(ABC): """ Interface between a particular parallel computing framework and xarray. @@ -190,7 +179,7 @@ class ChunkManagerEntrypoint(ABC, Generic[T_ChunkedArray]): This attribute is used for array instance type checking at runtime. """ - array_cls: type[T_ChunkedArray] + array_cls: type[chunkedduckarray[Any, Any]] available: bool = True @abstractmethod @@ -216,10 +205,10 @@ def is_chunked_array(self, data: duckarray[Any, Any]) -> bool: -------- dask.is_dask_collection """ - return isinstance(data, self.array_cls) + return isinstance(data, _chunkedarrayfunction_or_api) @abstractmethod - def chunks(self, data: T_ChunkedArray) -> _NormalizedChunks: + def chunks(self, data: chunkedduckarray[Any, Any]) -> _Chunks: """ Return the current chunks of the given array. @@ -245,12 +234,12 @@ def chunks(self, data: T_ChunkedArray) -> _NormalizedChunks: @abstractmethod def normalize_chunks( self, - chunks: _Chunks | _NormalizedChunks, - shape: _ShapeType | None = None, + chunks: _ChunksLike, + shape: _Shape | None = None, limit: int | None = None, dtype: _DType | None = None, - previous_chunks: _NormalizedChunks | None = None, - ) -> _NormalizedChunks: + previous_chunks: _Chunks | None = None, + ) -> _Chunks: """ Normalize given chunking pattern into an explicit tuple of tuples representation. @@ -281,8 +270,8 @@ def normalize_chunks( @abstractmethod def from_array( - self, data: duckarray[Any, Any], chunks: _Chunks, **kwargs: Any - ) -> T_ChunkedArray: + self, data: duckarray[Any, _DType], chunks: _ChunksLike, **kwargs: Any + ) -> chunkedduckarray[Any, _DType]: """ Create a chunked array from a non-chunked numpy-like array. @@ -307,10 +296,10 @@ def from_array( def rechunk( self, - data: T_ChunkedArray, - chunks: _NormalizedChunks | tuple[int, ...] | _Chunks, + data: chunkedduckarray[Any, _DType], + chunks: _ChunksLike, **kwargs: Any, - ) -> Any: + ) -> chunkedduckarray[Any, _DType]: """ Changes the chunking pattern of the given array. @@ -338,8 +327,8 @@ def rechunk( @abstractmethod def compute( - self, *data: T_ChunkedArray | Any, **kwargs: Any - ) -> tuple[np.ndarray[Any, _DType_co], ...]: + self, *data: chunkedduckarray[Any, _DType] | Any, **kwargs: Any + ) -> tuple[np.ndarray[Any, np.dtype[np.generic]], ...]: """ Computes one or more chunked arrays, returning them as eager numpy arrays. @@ -382,14 +371,14 @@ def array_api(self) -> Any: def reduction( self, - arr: T_ChunkedArray, + arr: chunkedduckarray[Any, _DType], func: Callable[..., Any], combine_func: Callable[..., Any] | None = None, aggregate_func: Callable[..., Any] | None = None, axis: int | Sequence[int] | None = None, - dtype: _DType_co | None = None, + dtype: _DType | None = None, keepdims: bool = False, - ) -> T_ChunkedArray: + ) -> chunkedduckarray[Any, _DType]: """ A general version of array reductions along one or more axes. @@ -434,11 +423,11 @@ def scan( func: Callable[..., Any], binop: Callable[..., Any], ident: float, - arr: T_ChunkedArray, + arr: chunkedduckarray[Any, _DType], axis: int | None = None, - dtype: _DType_co | None = None, + dtype: _DType | None = None, **kwargs: Any, - ) -> T_ChunkedArray: + ) -> chunkedduckarray[Any, _DType]: """ General version of a 1D scan, also known as a cumulative array reduction. @@ -474,10 +463,10 @@ def apply_gufunc( *args: Any, axes: Sequence[tuple[int, ...]] | None = None, keepdims: bool = False, - output_dtypes: Sequence[_DType_co] | None = None, + output_dtypes: Sequence[_DType] | None = None, vectorize: bool | None = None, **kwargs: Any, - ) -> Any: + ) -> chunkedduckarray[Any, _DType] | tuple[chunkedduckarray[Any, _DType], ...]: """ Apply a generalized ufunc or similar python function to arrays. @@ -557,12 +546,12 @@ def map_blocks( self, func: Callable[..., Any], *args: Any, - dtype: _DType_co | None = None, - chunks: tuple[int, ...] | None = None, + dtype: _DType | None = None, + chunks: _Chunks | None = None, drop_axis: int | Sequence[int] | None = None, new_axis: int | Sequence[int] | None = None, **kwargs: Any, - ) -> Any: + ) -> chunkedduckarray[Any, _DType]: """ Map a function across all blocks of a chunked array. @@ -610,7 +599,7 @@ def blockwise( new_axes: dict[Any, int] | None = None, align_arrays: bool = True, **kwargs: Any, - ) -> Any: + ) -> chunkedduckarray[Any, _DType]: """ Tensor operation: Generalized inner and outer products. @@ -656,7 +645,7 @@ def unify_chunks( self, *args: Any, # can't type this as mypy assumes args are all same type, but dask unify_chunks args alternate types **kwargs: Any, - ) -> tuple[dict[str, _NormalizedChunks], list[T_ChunkedArray]]: + ) -> tuple[dict[str, _Chunks], list[chunkedduckarray[Any, Any]]]: """ Unify chunks across a sequence of arrays. @@ -676,7 +665,9 @@ def unify_chunks( def store( self, - sources: T_ChunkedArray | Sequence[T_ChunkedArray], + sources: ( + chunkedduckarray[Any, _DType] | Sequence[chunkedduckarray[Any, _DType]] + ), targets: Any, **kwargs: dict[str, Any], ) -> Any: From 937723f8c1594bd73fa33bb809d9670f3c0df94b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Apr 2024 19:25:51 +0000 Subject: [PATCH 13/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/daskmanager.py | 9 ++------- xarray/namedarray/parallelcompat.py | 11 +++-------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index 6af8cd82252..518173adb6e 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any, Callable, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable, TypeVar import numpy as np from packaging.version import Version @@ -12,16 +12,11 @@ if TYPE_CHECKING: from xarray.namedarray._typing import ( - T_Chunks, _Chunks, _ChunksLike, - _ChunksType, - _dtype, _DType, - _DType_co, - _NormalizedChunks, + _dtype, _Shape, - _ShapeType, chunkedduckarray, duckarray, ) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index 0abea576180..54541e76a66 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -11,7 +11,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable, Sequence from importlib.metadata import EntryPoint, entry_points -from typing import TYPE_CHECKING, Any, Callable, Generic, Protocol, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable import numpy as np @@ -20,18 +20,13 @@ if TYPE_CHECKING: from xarray.namedarray._typing import ( + _chunkedarrayfunction_or_api, _Chunks, _ChunksLike, - _ChunksType, _DType, - _DType_co, - _NormalizedChunks, _Shape, - _ShapeType, - _ShapeType_co, - duckarray, - _chunkedarrayfunction_or_api, chunkedduckarray, + duckarray, ) From e83da1f487cccf800714c3efbd72ad1810b22d21 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 14 Apr 2024 21:51:07 +0200 Subject: [PATCH 14/47] more --- xarray/coding/times.py | 13 +++++---- xarray/namedarray/_typing.py | 2 +- xarray/namedarray/daskmanager.py | 42 ++++++++++++++++++++--------- xarray/namedarray/parallelcompat.py | 3 ++- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 466e847e003..7c48a8c9e03 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -27,9 +27,10 @@ from xarray.core.pdcompat import nanosecond_precision_timestamp from xarray.core.utils import emit_user_level_warning from xarray.core.variable import Variable -from xarray.namedarray.parallelcompat import T_ChunkedArray, get_chunked_array_type +from xarray.namedarray.parallelcompat import get_chunked_array_type from xarray.namedarray.pycompat import is_chunked_array from xarray.namedarray.utils import is_duck_dask_array +from xarray.namedarray._typing import chunkedduckarray try: import cftime @@ -816,11 +817,11 @@ def _encode_cf_datetime_within_map_blocks( def _lazily_encode_cf_datetime( - dates: T_ChunkedArray, + dates: chunkedduckarray, units: str | None = None, calendar: str | None = None, dtype: np.dtype | None = None, -) -> tuple[T_ChunkedArray, str, str]: +) -> tuple[chunkedduckarray, str, str]: if calendar is None: # This will only trigger minor compute if dates is an object dtype array. calendar = infer_calendar_name(dates) @@ -929,8 +930,10 @@ def _encode_cf_timedelta_within_map_blocks( def _lazily_encode_cf_timedelta( - timedeltas: T_ChunkedArray, units: str | None = None, dtype: np.dtype | None = None -) -> tuple[T_ChunkedArray, str]: + timedeltas: chunkedduckarray, + units: str | None = None, + dtype: np.dtype | None = None, +) -> tuple[chunkedduckarray, str]: if units is None and dtype is None: units = "nanoseconds" dtype = np.dtype("int64") diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 2fae69710a9..258ff924870 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -104,7 +104,7 @@ def dtype(self) -> _DType_co: ... _Chunks = tuple[_Chunk, ...] _NormalizedChunks = tuple[tuple[int, ...], ...] # TODO: Same as Chunks. _ChunksLike = Union[ - int, Literal["auto"], None, _chunk, _Chunks + int, Literal["auto"], None, _Chunk, _Chunks ] # TODO: Literal["auto"] _ChunksType = TypeVar("_ChunksType", bound=_Chunks) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index 518173adb6e..037b6ee7802 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Iterable, Sequence +from types import ModuleType from typing import TYPE_CHECKING, Any, Callable, TypeVar import numpy as np @@ -32,7 +33,7 @@ _T_DaskArray = TypeVar("_T_DaskArray", bound="DaskArray") -class DaskManager(ChunkManagerEntrypoint): # type: ignore[type-var] +class DaskManager(ChunkManagerEntrypoint): # array_cls: type[DaskArray] available: bool = dask_available @@ -47,26 +48,28 @@ def is_chunked_array(self, data: duckarray[Any, Any]) -> bool: return is_duck_dask_array(data) def chunks(self, data: chunkedduckarray[Any, Any]) -> _Chunks: - return data.chunks # type: ignore[no-any-return] + return data.chunks def normalize_chunks( self, chunks: _ChunksLike, shape: _Shape | None = None, limit: int | None = None, - dtype: _dtype | None = None, + dtype: _dtype[Any] | None = None, previous_chunks: _Chunks | None = None, ) -> _Chunks: """Called by open_dataset""" from dask.array.core import normalize_chunks - return normalize_chunks( + out: _Chunks + out = normalize_chunks( chunks, shape=shape, limit=limit, dtype=dtype, previous_chunks=previous_chunks, ) # type: ignore[no-untyped-call] + return out def from_array( self, data: duckarray[Any, _DType], chunks: _ChunksLike, **kwargs: Any @@ -77,11 +80,13 @@ def from_array( # lazily loaded backend array classes should use NumPy array operations. kwargs["meta"] = np.ndarray - return from_array( + out: chunkedduckarray[Any, _DType] + out = from_array( data, chunks, **kwargs, ) # type: ignore[no-untyped-call] + return out def compute( self, *data: chunkedduckarray[Any, _DType] | Any, **kwargs: Any @@ -91,7 +96,7 @@ def compute( return compute(*data, **kwargs) # type: ignore[no-untyped-call, no-any-return] @property - def array_api(self) -> Any: + def array_api(self) -> ModuleType: from dask import array as da return da @@ -108,7 +113,8 @@ def reduction( ) -> chunkedduckarray[Any, _DType]: from dask.array.reductions import reduction - return reduction( + out: chunkedduckarray[Any, _DType] + out = reduction( arr, chunk=func, combine=combine_func, @@ -117,6 +123,7 @@ def reduction( dtype=dtype, keepdims=keepdims, ) # type: ignore[no-untyped-call] + return out def scan( self, @@ -130,7 +137,8 @@ def scan( ) -> chunkedduckarray[Any, _DType]: from dask.array.reductions import cumreduction - return cumreduction( + out: chunkedduckarray[Any, _DType] + out = cumreduction( func, binop, ident, @@ -139,6 +147,7 @@ def scan( dtype=dtype, **kwargs, ) # type: ignore[no-untyped-call] + return out def apply_gufunc( self, @@ -157,7 +166,8 @@ def apply_gufunc( ) -> chunkedduckarray[Any, _DType] | tuple[chunkedduckarray[Any, _DType], ...]: from dask.array.gufunc import apply_gufunc - return apply_gufunc( + out: chunkedduckarray[Any, _DType] | tuple[chunkedduckarray[Any, _DType], ...] + out = apply_gufunc( func, signature, *args, @@ -172,6 +182,8 @@ def apply_gufunc( **kwargs, ) # type: ignore[no-untyped-call] + return out + def map_blocks( self, func: Callable[..., Any], @@ -191,7 +203,8 @@ def map_blocks( drop_axis = [] # pass through name, meta, token as kwargs - return map_blocks( + out: chunkedduckarray[Any, _DType] + out = map_blocks( func, *args, dtype=dtype, @@ -200,6 +213,7 @@ def map_blocks( new_axis=new_axis, **kwargs, ) # type: ignore[no-untyped-call] + return out def blockwise( self, @@ -218,7 +232,8 @@ def blockwise( ) -> chunkedduckarray[Any, _DType]: from dask.array.blockwise import blockwise - return blockwise( + out: chunkedduckarray[Any, _DType] + out = blockwise( func, out_ind, *args, @@ -232,6 +247,7 @@ def blockwise( meta=meta, **kwargs, ) # type: ignore[no-untyped-call] + return out def unify_chunks( self, @@ -240,7 +256,9 @@ def unify_chunks( ) -> tuple[dict[str, _Chunks], list[chunkedduckarray[Any, Any]]]: from dask.array.core import unify_chunks - return unify_chunks(*args, **kwargs) # type: ignore[no-any-return, no-untyped-call] + out: tuple[dict[str, _Chunks], list[chunkedduckarray[Any, Any]]] + out = unify_chunks(*args, **kwargs) # type: ignore[no-untyped-call] + return out def store( self, diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index 54541e76a66..a8889beda05 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -11,6 +11,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable, Sequence from importlib.metadata import EntryPoint, entry_points +from types import ModuleType from typing import TYPE_CHECKING, Any, Callable import numpy as np @@ -349,7 +350,7 @@ def compute( raise NotImplementedError() @property - def array_api(self) -> Any: + def array_api(self) -> ModuleType: """ Return the array_api namespace following the python array API standard. From 5c4ecab0c02a7660980ba49a1b8aa159f9a51838 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Apr 2024 19:51:41 +0000 Subject: [PATCH 15/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/coding/times.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 7c48a8c9e03..c420a9cd261 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -27,10 +27,10 @@ from xarray.core.pdcompat import nanosecond_precision_timestamp from xarray.core.utils import emit_user_level_warning from xarray.core.variable import Variable +from xarray.namedarray._typing import chunkedduckarray from xarray.namedarray.parallelcompat import get_chunked_array_type from xarray.namedarray.pycompat import is_chunked_array from xarray.namedarray.utils import is_duck_dask_array -from xarray.namedarray._typing import chunkedduckarray try: import cftime From c2fb0355b20d666d410b5f75d03df2135cd6c738 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 14 Apr 2024 22:32:12 +0200 Subject: [PATCH 16/47] Update daskmanager.py --- xarray/namedarray/daskmanager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index 037b6ee7802..6b932d9ce84 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -90,10 +90,12 @@ def from_array( def compute( self, *data: chunkedduckarray[Any, _DType] | Any, **kwargs: Any - ) -> tuple[np.ndarray[Any, np.dtype[np.generic]], ...]: + ) -> tuple[duckarray[Any, _DType], ...]: from dask.base import compute - return compute(*data, **kwargs) # type: ignore[no-untyped-call, no-any-return] + out: tuple[np.ndarray[Any, np.dtype[np.generic]], ...] + out = compute(*data, **kwargs) # type: ignore[no-untyped-call] + return out @property def array_api(self) -> ModuleType: From 6b525aefeef63b2e2715e0ddd6c972f69a79c90e Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 14 Apr 2024 22:32:59 +0200 Subject: [PATCH 17/47] Update parallelcompat.py --- xarray/namedarray/parallelcompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index a8889beda05..252ea5778ba 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -324,7 +324,7 @@ def rechunk( @abstractmethod def compute( self, *data: chunkedduckarray[Any, _DType] | Any, **kwargs: Any - ) -> tuple[np.ndarray[Any, np.dtype[np.generic]], ...]: + ) -> tuple[duckarray[Any, _DType], ...]: """ Computes one or more chunked arrays, returning them as eager numpy arrays. From 587a73a673043c10b2da323c67639c0d6bbc77eb Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 14 Apr 2024 23:08:56 +0200 Subject: [PATCH 18/47] Update daskmanager.py --- xarray/namedarray/daskmanager.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index 6b932d9ce84..f862a51d60f 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -30,11 +30,8 @@ dask_available = module_available("dask") -_T_DaskArray = TypeVar("_T_DaskArray", bound="DaskArray") - class DaskManager(ChunkManagerEntrypoint): - # array_cls: type[DaskArray] available: bool = dask_available def __init__(self) -> None: @@ -42,12 +39,13 @@ def __init__(self) -> None: from dask.array.core import Array - self.array_cls = Array + # TODO: error: Incompatible types in assignment (expression has type "type[Array]", variable has type "type[_chunkedarrayfunction[Any, Any]] | type[_chunkedarrayapi[Any, Any]]") [assignment] + self.array_cls = Array # type: ignore[assignment] def is_chunked_array(self, data: duckarray[Any, Any]) -> bool: return is_duck_dask_array(data) - def chunks(self, data: chunkedduckarray[Any, Any]) -> _Chunks: + def chunks(self, data: chunkedduckarray[Any, _dtype[Any]]) -> _Chunks: return data.chunks def normalize_chunks( @@ -93,7 +91,7 @@ def compute( ) -> tuple[duckarray[Any, _DType], ...]: from dask.base import compute - out: tuple[np.ndarray[Any, np.dtype[np.generic]], ...] + out: tuple[duckarray[Any, _DType], ...] out = compute(*data, **kwargs) # type: ignore[no-untyped-call] return out From f25581a2313cb8000c57aa9a9966a765be54acf6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Apr 2024 21:09:28 +0000 Subject: [PATCH 19/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/daskmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index f862a51d60f..fc1ba8211ab 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -2,7 +2,7 @@ from collections.abc import Iterable, Sequence from types import ModuleType -from typing import TYPE_CHECKING, Any, Callable, TypeVar +from typing import TYPE_CHECKING, Any, Callable import numpy as np from packaging.version import Version From 549233666b099319206f04364c579390e01849fd Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Wed, 15 May 2024 21:41:09 +0200 Subject: [PATCH 20/47] test --- xarray/namedarray/_typing.py | 36 ++++++++++++++++++++++++----------- xarray/namedarray/pycompat.py | 14 +++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 258ff924870..7470b02463f 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -70,6 +70,7 @@ def __eq__(self, other: _DType2[_generic], /) -> bool: _dtype = _DType2 _DType = TypeVar("_DType", bound=_dtype[_generic]) _DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[_generic]) +_DType_np = TypeVar("_DType_np", bound=np.dtype[np.generic]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` _ScalarType = TypeVar("_ScalarType", bound=_generic) @@ -171,21 +172,34 @@ def __getitem__( @overload def __getitem__(self, key: _IndexKeyLike, /) -> Any: ... - - def __getitem__( - self, - key: ( - _IndexKeyLike - | _arrayfunction[Any, Any] - | tuple[_arrayfunction[Any, Any], ...] - ), - /, - ) -> _arrayfunction[Any, _DType_co] | Any: ... + @overload + def __getitem__(self: Any, key: Any, /) -> Any: ... + + # @overload + # def __getitem__(self: NDArray[void], key: str) -> NDArray[Any]: ... + # @overload + # def __getitem__( + # self: NDArray[void], key: list[str] + # ) -> ndarray[_ShapeType, _dtype[void]]: ... + + # def __getitem__( + # self, + # key: ( + # _IndexKeyLike + # | _arrayfunction[Any, Any] + # | tuple[_arrayfunction[Any, Any], ...] + # ), + # /, + # ) -> _arrayfunction[Any, _DType_co] | Any: ... def __array__( - self, dtype: _dtype[_generic] | None = ..., / + self, dtype: Any | None = ..., / ) -> np.ndarray[Any, np.dtype[np.generic]]: ... + # def __array__( + # self, dtype: _dtype[_generic] | None = ..., / + # ) -> np.ndarray[Any, np.dtype[np.generic]]: ... + # TODO: Should return the same subclass but with a new dtype generic. # https://github.com/python/typing/issues/548 def __array_ufunc__( diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 7d32c8380b6..cff39bdfcaa 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -139,12 +139,12 @@ def to_duck_array( ) -> chunkedduckarray[_ShapeType, _DType]: ... @overload def to_duck_array( - data: duckarray[_ShapeType, _DType], **kwargs: dict[str, Any] -) -> duckarray[_ShapeType, _DType]: ... + data: np.typing.ArrayLike, **kwargs: dict[str, Any] +) -> np.ndarray[Any, np.dtype[np.generic]]: ... @overload def to_duck_array( - data: np.typing.ArrayLike, **kwargs: dict[str, Any] -) -> duckarray[Any, _dtype[_generic]]: ... + data: duckarray[_ShapeType, _DType], **kwargs: dict[str, Any] +) -> duckarray[_ShapeType, _DType]: ... def to_duck_array( data: ( ExplicitlyIndexed @@ -153,7 +153,11 @@ def to_duck_array( | np.typing.ArrayLike ), **kwargs: dict[str, Any], -) -> duckarray[_ShapeType, _DType] | duckarray[Any, _dtype[_generic]]: +) -> ( + duckarray[_ShapeType, _DType] + | duckarray[Any, _dtype[_generic]] + | np.ndarray[Any, np.dtype[np.generic]] +): from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.parallelcompat import get_chunked_array_type From fb7ccfa995021ef75fee830dbd85bd6e6a6930ef Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Wed, 15 May 2024 21:57:49 +0200 Subject: [PATCH 21/47] Update pycompat.py --- xarray/namedarray/pycompat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index cff39bdfcaa..f04e290344e 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -124,9 +124,9 @@ def to_numpy( data = data.magnitude if isinstance(data, array_type("sparse")): data = data.todense() - data = np.asarray(data) + out = np.asarray(data) - return data + return out @overload From 9140f36fd86c6bad3c47210e4ec12b21572d8d25 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 7 Jun 2024 10:40:28 +0200 Subject: [PATCH 22/47] Update daskmanager.py --- xarray/namedarray/daskmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/daskmanager.py b/xarray/namedarray/daskmanager.py index fc1ba8211ab..cfd788793e1 100644 --- a/xarray/namedarray/daskmanager.py +++ b/xarray/namedarray/daskmanager.py @@ -89,7 +89,7 @@ def from_array( def compute( self, *data: chunkedduckarray[Any, _DType] | Any, **kwargs: Any ) -> tuple[duckarray[Any, _DType], ...]: - from dask.base import compute + from dask.array import compute out: tuple[duckarray[Any, _DType], ...] out = compute(*data, **kwargs) # type: ignore[no-untyped-call] From bbfbca315f34d19999d0445bde263036f724918e Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:22:00 +0200 Subject: [PATCH 23/47] Update pycompat.py --- xarray/namedarray/pycompat.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index f04e290344e..16136b632c3 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -161,14 +161,15 @@ def to_duck_array( from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.parallelcompat import get_chunked_array_type + if isinstance(data, ExplicitlyIndexed): + return data.get_duck_array() # type: ignore[no-untyped-call, no-any-return] + if is_chunked_array(data): chunkmanager = get_chunked_array_type(data) loaded_data, *_ = chunkmanager.compute(data, **kwargs) # type: ignore[var-annotated] return loaded_data - if isinstance(data, ExplicitlyIndexed): - return data.get_duck_array() # type: ignore[no-untyped-call, no-any-return] - elif is_duck_array(data): + if is_duck_array(data): return data else: - return np.asarray(data) # type: ignore[return-value] + return np.asarray(data) From e57d85d6829949140af576405dc7c9856aa09cd6 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:48:37 +0200 Subject: [PATCH 24/47] Update pycompat.py --- xarray/namedarray/pycompat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 16136b632c3..3761a857e4b 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -138,13 +138,13 @@ def to_duck_array( data: chunkedduckarray[_ShapeType, _DType], **kwargs: dict[str, Any] ) -> chunkedduckarray[_ShapeType, _DType]: ... @overload -def to_duck_array( - data: np.typing.ArrayLike, **kwargs: dict[str, Any] -) -> np.ndarray[Any, np.dtype[np.generic]]: ... -@overload def to_duck_array( data: duckarray[_ShapeType, _DType], **kwargs: dict[str, Any] ) -> duckarray[_ShapeType, _DType]: ... +@overload +def to_duck_array( + data: np.typing.ArrayLike, **kwargs: dict[str, Any] +) -> np.ndarray[Any, np.dtype[np.generic]]: ... def to_duck_array( data: ( ExplicitlyIndexed From d6019422d5c0f6a733e1059dd13b90a1e9fdacd8 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:07:06 +0200 Subject: [PATCH 25/47] Update pycompat.py --- xarray/namedarray/pycompat.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 3761a857e4b..2f22fb87795 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -141,10 +141,16 @@ def to_duck_array( def to_duck_array( data: duckarray[_ShapeType, _DType], **kwargs: dict[str, Any] ) -> duckarray[_ShapeType, _DType]: ... + + +# @overload +# def to_duck_array( +# data: np.typing.ArrayLike, **kwargs: dict[str, Any] +# ) -> np.ndarray[Any, np.dtype[np.generic]]: ... @overload def to_duck_array( data: np.typing.ArrayLike, **kwargs: dict[str, Any] -) -> np.ndarray[Any, np.dtype[np.generic]]: ... +) -> duckarray[Any, np.dtype[np.generic]]: ... def to_duck_array( data: ( ExplicitlyIndexed From b8da922e6da76113f98006c4d8f62ed9b8b12c42 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:14:17 +0200 Subject: [PATCH 26/47] Update pycompat.py --- xarray/namedarray/pycompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 2f22fb87795..616c55ea5ee 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -150,7 +150,7 @@ def to_duck_array( @overload def to_duck_array( data: np.typing.ArrayLike, **kwargs: dict[str, Any] -) -> duckarray[Any, np.dtype[np.generic]]: ... +) -> duckarray[Any, _dtype[_generic]]: ... def to_duck_array( data: ( ExplicitlyIndexed From 7222ae47eb947063d5cb3835a3d80f81d36ada2a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 17:53:26 +0200 Subject: [PATCH 27/47] Let's try undoing --- xarray/namedarray/_typing.py | 48 +++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 7470b02463f..3aeb4505973 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -12,6 +12,7 @@ Literal, Protocol, SupportsIndex, + TypeAlias, TypeVar, Union, overload, @@ -43,7 +44,8 @@ class Default(Enum): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) -_generic = Any +_generic: TypeAlias = Any +# _generic = np.generic class _DType2(Protocol[_T_co]): @@ -66,15 +68,17 @@ def __eq__(self, other: _DType2[_generic], /) -> bool: ... -# _dtype = np.dtype -_dtype = _DType2 +_dtype = np.dtype +# _dtype = _DType2 _DType = TypeVar("_DType", bound=_dtype[_generic]) _DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[_generic]) _DType_np = TypeVar("_DType_np", bound=np.dtype[np.generic]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` -_ScalarType = TypeVar("_ScalarType", bound=_generic) -_ScalarType_co = TypeVar("_ScalarType_co", bound=_generic, covariant=True) +# _ScalarType = TypeVar("_ScalarType", bound=_generic) +# _ScalarType_co = TypeVar("_ScalarType_co", bound=_generic, covariant=True) +_ScalarType = TypeVar("_ScalarType", bound=np.generic) +_ScalarType_co = TypeVar("_ScalarType_co", bound=np.generic, covariant=True) # A protocol for anything with the dtype attribute @@ -172,8 +176,21 @@ def __getitem__( @overload def __getitem__(self, key: _IndexKeyLike, /) -> Any: ... - @overload - def __getitem__(self: Any, key: Any, /) -> Any: ... + + def __getitem__( + self, + key: ( + _IndexKeyLike + | _arrayfunction[Any, Any] + | tuple[_arrayfunction[Any, Any], ...] + ), + /, + ) -> _arrayfunction[Any, _DType_co] | Any: ... + + # @overload + # def __getitem__(self, key: _IndexKeyLike, /) -> Any: ... + # @overload + # def __getitem__(self: Any, key: Any, /) -> Any: ... # @overload # def __getitem__(self: NDArray[void], key: str) -> NDArray[Any]: ... @@ -182,16 +199,6 @@ def __getitem__(self: Any, key: Any, /) -> Any: ... # self: NDArray[void], key: list[str] # ) -> ndarray[_ShapeType, _dtype[void]]: ... - # def __getitem__( - # self, - # key: ( - # _IndexKeyLike - # | _arrayfunction[Any, Any] - # | tuple[_arrayfunction[Any, Any], ...] - # ), - # /, - # ) -> _arrayfunction[Any, _DType_co] | Any: ... - def __array__( self, dtype: Any | None = ..., / ) -> np.ndarray[Any, np.dtype[np.generic]]: ... @@ -365,3 +372,10 @@ def todense(self) -> np.ndarray[Any, np.dtype[np.generic]]: ... ErrorOptions = Literal["raise", "ignore"] ErrorOptionsWithWarn = Literal["raise", "warn", "ignore"] + + +def test(arr: duckarray[_ShapeType, _DType]) -> duckarray[_ShapeType, _DType]: + return arr + + +test(np.array([], dtype=np.int64)) From dc784eb0fb72c440ce25da7e6c8e7993ae33c5da Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 17:56:21 +0200 Subject: [PATCH 28/47] Update _typing.py --- xarray/namedarray/_typing.py | 60 +++++++----------------------------- 1 file changed, 11 insertions(+), 49 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index 3aeb4505973..de8cb037f89 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -44,39 +44,11 @@ class Default(Enum): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) -_generic: TypeAlias = Any -# _generic = np.generic - - -class _DType2(Protocol[_T_co]): - def __eq__(self, other: _DType2[_generic], /) -> bool: - """ - Computes the truth value of ``self == other`` in order to test for data type object equality. - - Parameters - ---------- - self: dtype - data type instance. May be any supported data type. - other: dtype - other data type instance. May be any supported data type. - - Returns - ------- - out: bool - a boolean indicating whether the data type objects are equal. - """ - ... - - _dtype = np.dtype -# _dtype = _DType2 -_DType = TypeVar("_DType", bound=_dtype[_generic]) -_DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[_generic]) -_DType_np = TypeVar("_DType_np", bound=np.dtype[np.generic]) +_DType = TypeVar("_DType", bound=np.dtype[Any]) +_DType_co = TypeVar("_DType_co", covariant=True, bound=np.dtype[Any]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` -# _ScalarType = TypeVar("_ScalarType", bound=_generic) -# _ScalarType_co = TypeVar("_ScalarType_co", bound=_generic, covariant=True) _ScalarType = TypeVar("_ScalarType", bound=np.generic) _ScalarType_co = TypeVar("_ScalarType_co", bound=np.generic, covariant=True) @@ -89,9 +61,9 @@ def dtype(self) -> _DType_co: ... _DTypeLike = Union[ - _dtype[_ScalarType], + np.dtype[_ScalarType], type[_ScalarType], - _SupportsDType[_dtype[_ScalarType]], + _SupportsDType[np.dtype[_ScalarType]], ] # For unknown shapes Dask uses np.nan, array_api uses None: @@ -187,25 +159,15 @@ def __getitem__( /, ) -> _arrayfunction[Any, _DType_co] | Any: ... - # @overload - # def __getitem__(self, key: _IndexKeyLike, /) -> Any: ... - # @overload - # def __getitem__(self: Any, key: Any, /) -> Any: ... + @overload + def __array__(self, dtype: None = ..., /) -> np.ndarray[Any, _DType_co]: ... - # @overload - # def __getitem__(self: NDArray[void], key: str) -> NDArray[Any]: ... - # @overload - # def __getitem__( - # self: NDArray[void], key: list[str] - # ) -> ndarray[_ShapeType, _dtype[void]]: ... + @overload + def __array__(self, dtype: _DType, /) -> np.ndarray[Any, _DType]: ... def __array__( - self, dtype: Any | None = ..., / - ) -> np.ndarray[Any, np.dtype[np.generic]]: ... - - # def __array__( - # self, dtype: _dtype[_generic] | None = ..., / - # ) -> np.ndarray[Any, np.dtype[np.generic]]: ... + self, dtype: _DType | None = ..., / + ) -> np.ndarray[Any, _DType] | np.ndarray[Any, _DType_co]: ... # TODO: Should return the same subclass but with a new dtype generic. # https://github.com/python/typing/issues/548 @@ -261,7 +223,7 @@ def __array_namespace__(self) -> ModuleType: ... ] # Corresponds to np.typing.NDArray: -DuckArray = _arrayfunction[Any, _dtype[_ScalarType_co]] +DuckArray = _arrayfunction[Any, np.dtype[_ScalarType_co]] @runtime_checkable From ce879ca007a19591bd8bb9b3e9ca5965cd59f87e Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:06:47 +0200 Subject: [PATCH 29/47] Update _typing.py --- xarray/namedarray/_typing.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index de8cb037f89..d7e8d4a5b51 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -44,11 +44,37 @@ class Default(Enum): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) +_generic: TypeAlias = Any +# _generic = np.generic + + +class _DType2(Protocol[_T_co]): + def __eq__(self, other: _DType2[_generic], /) -> bool: + """ + Computes the truth value of ``self == other`` in order to test for data type object equality. + + Parameters + ---------- + self: dtype + data type instance. May be any supported data type. + other: dtype + other data type instance. May be any supported data type. + + Returns + ------- + out: bool + a boolean indicating whether the data type objects are equal. + """ + ... + + _dtype = np.dtype _DType = TypeVar("_DType", bound=np.dtype[Any]) _DType_co = TypeVar("_DType_co", covariant=True, bound=np.dtype[Any]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` +# _ScalarType = TypeVar("_ScalarType", bound=_generic) +# _ScalarType_co = TypeVar("_ScalarType_co", bound=_generic, covariant=True) _ScalarType = TypeVar("_ScalarType", bound=np.generic) _ScalarType_co = TypeVar("_ScalarType_co", bound=np.generic, covariant=True) From 6cc75cc8b5bf9bdfc2bc79be9ee973e93a57840b Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:13:11 +0200 Subject: [PATCH 30/47] Update _typing.py --- xarray/namedarray/_typing.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index d7e8d4a5b51..b052e5ee76e 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -12,7 +12,6 @@ Literal, Protocol, SupportsIndex, - TypeAlias, TypeVar, Union, overload, @@ -44,7 +43,7 @@ class Default(Enum): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) -_generic: TypeAlias = Any +_generic = Any # _generic = np.generic @@ -69,8 +68,8 @@ def __eq__(self, other: _DType2[_generic], /) -> bool: _dtype = np.dtype -_DType = TypeVar("_DType", bound=np.dtype[Any]) -_DType_co = TypeVar("_DType_co", covariant=True, bound=np.dtype[Any]) +_DType = TypeVar("_DType", bound=_dtype[Any]) +_DType_co = TypeVar("_DType_co", covariant=True, bound=_dtype[Any]) # A subset of `npt.DTypeLike` that can be parametrized w.r.t. `np.generic` # _ScalarType = TypeVar("_ScalarType", bound=_generic) @@ -87,9 +86,9 @@ def dtype(self) -> _DType_co: ... _DTypeLike = Union[ - np.dtype[_ScalarType], + _dtype[_ScalarType], type[_ScalarType], - _SupportsDType[np.dtype[_ScalarType]], + _SupportsDType[_dtype[_ScalarType]], ] # For unknown shapes Dask uses np.nan, array_api uses None: @@ -249,7 +248,7 @@ def __array_namespace__(self) -> ModuleType: ... ] # Corresponds to np.typing.NDArray: -DuckArray = _arrayfunction[Any, np.dtype[_ScalarType_co]] +DuckArray = _arrayfunction[Any, _dtype[_ScalarType_co]] @runtime_checkable From c2ee93cdde336b49f868c491427bcbc2a11704c3 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:20:44 +0200 Subject: [PATCH 31/47] chunkmanager doesn't use generic anymore --- xarray/core/indexing.py | 2 +- xarray/core/variable.py | 2 +- xarray/namedarray/core.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 06e7efdbb48..55d3c41acec 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -1325,7 +1325,7 @@ def _arrayize_vectorized_indexer( def _chunked_array_with_chunks_hint( - array, chunks, chunkmanager: ChunkManagerEntrypoint[Any] + array, chunks, chunkmanager: ChunkManagerEntrypoint ): """Create a chunked array using the chunks hint for dimensions of size > 1.""" diff --git a/xarray/core/variable.py b/xarray/core/variable.py index f0685882595..594c4287d4d 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -2526,7 +2526,7 @@ def chunk( # type: ignore[override] name: str | None = None, lock: bool | None = None, inline_array: bool | None = None, - chunked_array_type: str | ChunkManagerEntrypoint[Any] | None = None, + chunked_array_type: str | ChunkManagerEntrypoint | None = None, from_array_kwargs: Any = None, **chunks_kwargs: Any, ) -> Self: diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 317ca76d23c..0f8a91c6c8f 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -749,7 +749,7 @@ def sizes(self) -> dict[_Dim, _IntOrUnknown]: def chunk( self, chunks: int | Literal["auto"] | Mapping[Any, None | int | tuple[int, ...]] = {}, - chunked_array_type: str | ChunkManagerEntrypoint[Any] | None = None, + chunked_array_type: str | ChunkManagerEntrypoint | None = None, from_array_kwargs: Any = None, **chunks_kwargs: Any, ) -> Self: From 21f9049e60aafd50000e1d6e4adcf9b8f62cac4d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:35:55 +0200 Subject: [PATCH 32/47] Update core.py --- xarray/namedarray/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 0f8a91c6c8f..0c5fc1073e0 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -822,6 +822,7 @@ def chunk( chunkmanager = guess_chunkmanager(chunked_array_type) data_old = self._data + data_chunked: _chunkedarray[Any, _DType_co] if chunkmanager.is_chunked_array(data_old): data_chunked = chunkmanager.rechunk(data_old, chunks) # type: ignore[arg-type] else: @@ -990,7 +991,7 @@ def _to_dense(self) -> NamedArray[Any, _DType_co]: Change backend from sparse to np.array. """ if isinstance(self._data, _sparsearrayfunction_or_api): - data_dense: np.ndarray[Any, _DType_co] = self._data.todense() + data_dense: np.ndarray[Any, Any] = self._data.todense() return self._new(data=data_dense) else: raise TypeError("self.data is not a sparse array") From d2e360d53f3f3ce5df228ac2f4dc8bf712c21d4a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:17:26 +0200 Subject: [PATCH 33/47] fix --- xarray/coding/times.py | 10 +++++----- xarray/coding/variables.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index dd2446aee6e..8e05f74561b 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -27,7 +27,7 @@ from xarray.core.pdcompat import nanosecond_precision_timestamp from xarray.core.utils import emit_user_level_warning from xarray.core.variable import Variable -from xarray.namedarray._typing import chunkedduckarray +from xarray.namedarray._typing import chunkedduckarray, duckarray from xarray.namedarray.parallelcompat import get_chunked_array_type from xarray.namedarray.pycompat import is_chunked_array from xarray.namedarray.utils import is_duck_dask_array @@ -702,11 +702,11 @@ def _cast_to_dtype_if_safe(num: np.ndarray, dtype: np.dtype) -> np.ndarray: def encode_cf_datetime( - dates: T_DuckArray, # type: ignore + dates: duckarray, units: str | None = None, calendar: str | None = None, dtype: np.dtype | None = None, -) -> tuple[T_DuckArray, str, str]: +) -> tuple[duckarray, str, str]: """Given an array of datetime objects, returns the tuple `(num, units, calendar)` suitable for a CF compliant time variable. @@ -857,10 +857,10 @@ def _lazily_encode_cf_datetime( def encode_cf_timedelta( - timedeltas: T_DuckArray, # type: ignore + timedeltas: duckarray, units: str | None = None, dtype: np.dtype | None = None, -) -> tuple[T_DuckArray, str]: +) -> tuple[duckarray, str]: timedeltas = asarray(timedeltas) if is_chunked_array(timedeltas): return _lazily_encode_cf_timedelta(timedeltas, units, dtype) diff --git a/xarray/coding/variables.py b/xarray/coding/variables.py index d31cb6e626a..3e1104dc202 100644 --- a/xarray/coding/variables.py +++ b/xarray/coding/variables.py @@ -179,7 +179,7 @@ def lazy_elemwise_func(array, func: Callable, dtype: np.typing.DTypeLike): if is_chunked_array(array): chunkmanager = get_chunked_array_type(array) - return chunkmanager.map_blocks(func, array, dtype=dtype) # type: ignore[arg-type] + return chunkmanager.map_blocks(func, array, dtype=dtype) else: return _ElementwiseFunctionArray(array, func, dtype) From 46b02189724925e71701845eb8188620e00c1975 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:44:20 +0200 Subject: [PATCH 34/47] Update times.py --- xarray/coding/times.py | 51 ++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 8e05f74561b..11e0b96fc98 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -5,7 +5,7 @@ from collections.abc import Hashable from datetime import datetime, timedelta from functools import partial -from typing import TYPE_CHECKING, Callable, Union +from typing import TYPE_CHECKING, Callable, Union, overload import numpy as np import pandas as pd @@ -27,9 +27,14 @@ from xarray.core.pdcompat import nanosecond_precision_timestamp from xarray.core.utils import emit_user_level_warning from xarray.core.variable import Variable -from xarray.namedarray._typing import chunkedduckarray, duckarray +from xarray.namedarray._typing import ( + chunkedduckarray, + duckarray, + _chunkedarrayfunction_or_api, +) from xarray.namedarray.parallelcompat import get_chunked_array_type -from xarray.namedarray.pycompat import is_chunked_array + +# from xarray.namedarray.pycompat import is_chunked_array from xarray.namedarray.utils import is_duck_dask_array try: @@ -38,7 +43,7 @@ cftime = None if TYPE_CHECKING: - from xarray.core.types import CFCalendar, T_DuckArray + from xarray.core.types import CFCalendar T_Name = Union[Hashable, None] @@ -702,11 +707,11 @@ def _cast_to_dtype_if_safe(num: np.ndarray, dtype: np.dtype) -> np.ndarray: def encode_cf_datetime( - dates: duckarray, + dates: duckarray | chunkedduckarray, units: str | None = None, calendar: str | None = None, dtype: np.dtype | None = None, -) -> tuple[duckarray, str, str]: +) -> tuple[duckarray | chunkedduckarray, str, str]: """Given an array of datetime objects, returns the tuple `(num, units, calendar)` suitable for a CF compliant time variable. @@ -717,19 +722,19 @@ def encode_cf_datetime( cftime.date2num """ dates = asarray(dates) - if is_chunked_array(dates): + if isinstance(dates, _chunkedarrayfunction_or_api): return _lazily_encode_cf_datetime(dates, units, calendar, dtype) else: return _eagerly_encode_cf_datetime(dates, units, calendar, dtype) def _eagerly_encode_cf_datetime( - dates: T_DuckArray, # type: ignore + dates: duckarray, # type: ignore units: str | None = None, calendar: str | None = None, dtype: np.dtype | None = None, allow_units_modification: bool = True, -) -> tuple[T_DuckArray, str, str]: +) -> tuple[duckarray, str, str]: dates = asarray(dates) data_units = infer_datetime_units(dates) @@ -807,11 +812,11 @@ def _eagerly_encode_cf_datetime( def _encode_cf_datetime_within_map_blocks( - dates: T_DuckArray, # type: ignore + dates: duckarray, units: str, calendar: str, dtype: np.dtype, -) -> T_DuckArray: +) -> duckarray: num, *_ = _eagerly_encode_cf_datetime( dates, units, calendar, dtype, allow_units_modification=False ) @@ -856,24 +861,36 @@ def _lazily_encode_cf_datetime( return num, units, calendar +@overload +def encode_cf_timedelta( + timedeltas: chunkedduckarray, + units: str | None = None, + dtype: np.dtype | None = None, +) -> tuple[chunkedduckarray, str]: ... +@overload def encode_cf_timedelta( timedeltas: duckarray, units: str | None = None, dtype: np.dtype | None = None, -) -> tuple[duckarray, str]: +) -> tuple[duckarray, str]: ... +def encode_cf_timedelta( + timedeltas: chunkedduckarray | duckarray, + units: str | None = None, + dtype: np.dtype | None = None, +) -> tuple[chunkedduckarray | duckarray, str]: timedeltas = asarray(timedeltas) - if is_chunked_array(timedeltas): + if isinstance(timedeltas, _chunkedarrayfunction_or_api): return _lazily_encode_cf_timedelta(timedeltas, units, dtype) else: return _eagerly_encode_cf_timedelta(timedeltas, units, dtype) def _eagerly_encode_cf_timedelta( - timedeltas: T_DuckArray, # type: ignore + timedeltas: duckarray, units: str | None = None, dtype: np.dtype | None = None, allow_units_modification: bool = True, -) -> tuple[T_DuckArray, str]: +) -> tuple[duckarray, str]: data_units = infer_timedelta_units(timedeltas) if units is None: @@ -921,10 +938,10 @@ def _eagerly_encode_cf_timedelta( def _encode_cf_timedelta_within_map_blocks( - timedeltas: T_DuckArray, # type:ignore + timedeltas: duckarray, # type:ignore units: str, dtype: np.dtype, -) -> T_DuckArray: +) -> duckarray: num, _ = _eagerly_encode_cf_timedelta( timedeltas, units, dtype, allow_units_modification=False ) From c17adf3729b61569a0c8cf1753acda08b03b1115 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 17:45:01 +0000 Subject: [PATCH 35/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/coding/times.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 11e0b96fc98..a141133378d 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -28,9 +28,9 @@ from xarray.core.utils import emit_user_level_warning from xarray.core.variable import Variable from xarray.namedarray._typing import ( + _chunkedarrayfunction_or_api, chunkedduckarray, duckarray, - _chunkedarrayfunction_or_api, ) from xarray.namedarray.parallelcompat import get_chunked_array_type From 17d49e5ab4b50c6e1acecae0bfe9c359c6cd39e0 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:00:22 +0200 Subject: [PATCH 36/47] use duck_array_ops.ravel --- xarray/coding/times.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index a141133378d..e4c701febc1 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -22,7 +22,7 @@ ) from xarray.core import indexing from xarray.core.common import contains_cftime_datetimes, is_np_datetime_like -from xarray.core.duck_array_ops import asarray +from xarray.core.duck_array_ops import asarray, ravel from xarray.core.formatting import first_n_items, format_timestamp, last_item from xarray.core.pdcompat import nanosecond_precision_timestamp from xarray.core.utils import emit_user_level_warning @@ -321,7 +321,7 @@ def decode_cf_datetime( cftime.num2date """ num_dates = np.asarray(num_dates) - flat_num_dates = num_dates.ravel() + flat_num_dates = ravel(num_dates) if calendar is None: calendar = "standard" @@ -375,7 +375,7 @@ def decode_cf_timedelta(num_timedeltas, units: str) -> np.ndarray: """ num_timedeltas = np.asarray(num_timedeltas) units = _netcdf_to_numpy_timeunit(units) - result = to_timedelta_unboxed(num_timedeltas.ravel(), unit=units) + result = to_timedelta_unboxed(ravel(num_timedeltas), unit=units) return result.reshape(num_timedeltas.shape) @@ -434,7 +434,7 @@ def infer_datetime_units(dates) -> str: 'hours', 'minutes' or 'seconds' (the first one that can evenly divide all unique time deltas in `dates`) """ - dates = np.asarray(dates).ravel() + dates = ravel(np.asarray(dates)) if np.asarray(dates).dtype == "datetime64[ns]": dates = to_datetime_unboxed(dates) dates = dates[pd.notnull(dates)] @@ -462,7 +462,7 @@ def infer_timedelta_units(deltas) -> str: {'days', 'hours', 'minutes' 'seconds'} (the first one that can evenly divide all unique time deltas in `deltas`) """ - deltas = to_timedelta_unboxed(np.asarray(deltas).ravel()) + deltas = to_timedelta_unboxed(ravel(np.asarray(deltas))) unique_timedeltas = np.unique(deltas[pd.notnull(deltas)]) return _infer_time_units_from_diff(unique_timedeltas) @@ -649,7 +649,7 @@ def encode_datetime(d): except TypeError: return np.nan if d is None else cftime.date2num(d, units, calendar) - return np.array([encode_datetime(d) for d in dates.ravel()]).reshape(dates.shape) + return np.array([encode_datetime(d) for d in ravel(dates)]).reshape(dates.shape) def cast_to_int_if_safe(num) -> np.ndarray: @@ -729,7 +729,7 @@ def encode_cf_datetime( def _eagerly_encode_cf_datetime( - dates: duckarray, # type: ignore + dates: duckarray, units: str | None = None, calendar: str | None = None, dtype: np.dtype | None = None, @@ -759,7 +759,7 @@ def _eagerly_encode_cf_datetime( # Wrap the dates in a DatetimeIndex to do the subtraction to ensure # an OverflowError is raised if the ref_date is too far away from # dates to be encoded (GH 2272). - dates_as_index = pd.DatetimeIndex(dates.ravel()) + dates_as_index = pd.DatetimeIndex(ravel(dates)) time_deltas = dates_as_index - ref_date # retrieve needed units to faithfully encode to int64 @@ -897,7 +897,7 @@ def _eagerly_encode_cf_timedelta( units = data_units time_delta = _time_units_to_timedelta64(units) - time_deltas = pd.TimedeltaIndex(timedeltas.ravel()) + time_deltas = pd.TimedeltaIndex(ravel(timedeltas)) # retrieve needed units to faithfully encode to int64 needed_units = data_units From 7a659f24ec9191984b24713bb127a133d2a7f318 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:10:22 +0200 Subject: [PATCH 37/47] fix --- xarray/coding/times.py | 2 +- xarray/core/computation.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index e4c701febc1..a5f18496661 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -938,7 +938,7 @@ def _eagerly_encode_cf_timedelta( def _encode_cf_timedelta_within_map_blocks( - timedeltas: duckarray, # type:ignore + timedeltas: duckarray, units: str, dtype: np.dtype, ) -> duckarray: diff --git a/xarray/core/computation.py b/xarray/core/computation.py index f418d3821c2..a5d5ac09405 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -25,6 +25,7 @@ from xarray.core.types import Dims, T_DataArray from xarray.core.utils import is_dict_like, is_duck_dask_array, is_scalar, parse_dims from xarray.core.variable import Variable +from xarray.namedarray._typing import chunkedduckarray from xarray.namedarray.parallelcompat import get_chunked_array_type from xarray.namedarray.pycompat import is_chunked_array from xarray.util.deprecation_helpers import deprecate_dims @@ -795,6 +796,7 @@ def apply_variable_ufunc( ) def func(*arrays): + res: chunkedduckarray | tuple[chunkedduckarray, ...] res = chunkmanager.apply_gufunc( numpy_func, signature.to_gufunc_string(exclude_dims), From 7ddef25fad4ba9fe6b80f143e18396501275a36a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:06:58 +0200 Subject: [PATCH 38/47] Update dataset.py --- xarray/core/dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 50cfc7b0c29..7e0d0151d95 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -163,6 +163,7 @@ T_Xarray, ) from xarray.core.weighted import DatasetWeighted + from xarray.namedarray._typing import duckarray from xarray.namedarray.parallelcompat import ChunkManagerEntrypoint @@ -860,7 +861,7 @@ def load(self, **kwargs) -> Self: chunkmanager = get_chunked_array_type(*lazy_data.values()) # evaluate all the chunked arrays simultaneously - evaluated_data: tuple[np.ndarray[Any, Any], ...] = chunkmanager.compute( + evaluated_data: tuple[duckarray[Any, Any], ...] = chunkmanager.compute( *lazy_data.values(), **kwargs ) From 121774dc5cd1145d42fda5e0fcf87796c5e1157f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:19:17 +0200 Subject: [PATCH 39/47] Update test_parallelcompat.py --- xarray/tests/test_parallelcompat.py | 37 +++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index dbe40be710c..c46ef1c2fdd 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -6,8 +6,15 @@ import numpy as np import pytest -from xarray.core.types import T_Chunks, T_DuckArray, T_NormalizedChunks -from xarray.namedarray._typing import _Chunks +from xarray.core.types import T_DuckArray +from xarray.namedarray._typing import ( + _Chunks, + chunkedduckarray, + _ChunksLike, + _Shape, + _DType, + duckarray, +) from xarray.namedarray.daskmanager import DaskManager from xarray.namedarray.parallelcompat import ( ChunkManagerEntrypoint, @@ -27,7 +34,7 @@ class DummyChunkedArray(np.ndarray): https://numpy.org/doc/stable/user/basics.subclassing.html#simple-example-adding-an-extra-attribute-to-ndarray """ - chunks: T_NormalizedChunks + chunks: _Chunks def __new__( cls, @@ -63,32 +70,36 @@ def __init__(self): def is_chunked_array(self, data: Any) -> bool: return isinstance(data, DummyChunkedArray) - def chunks(self, data: DummyChunkedArray) -> T_NormalizedChunks: + def chunks(self, data: chunkedduckarray[Any, Any]) -> _Chunks: return data.chunks def normalize_chunks( self, - chunks: T_Chunks | T_NormalizedChunks, - shape: tuple[int, ...] | None = None, + chunks: _ChunksLike, + shape: _Shape | None = None, limit: int | None = None, - dtype: np.dtype | None = None, - previous_chunks: T_NormalizedChunks | None = None, - ) -> T_NormalizedChunks: + dtype: _DType | None = None, + previous_chunks: _Chunks | None = None, + ) -> _Chunks: from dask.array.core import normalize_chunks return normalize_chunks(chunks, shape, limit, dtype, previous_chunks) def from_array( - self, data: T_DuckArray | np.typing.ArrayLike, chunks: _Chunks, **kwargs - ) -> DummyChunkedArray: + self, data: duckarray[Any, _DType], chunks: _Chunks, **kwargs + ) -> chunkedduckarray[Any, _DType]: from dask import array as da return da.from_array(data, chunks, **kwargs) - def rechunk(self, data: DummyChunkedArray, chunks, **kwargs) -> DummyChunkedArray: + def rechunk( + self, data: chunkedduckarray[Any, _DType], chunks: _ChunksLike, **kwargs + ) -> chunkedduckarray[Any, _DType]: return data.rechunk(chunks, **kwargs) - def compute(self, *data: DummyChunkedArray, **kwargs) -> tuple[np.ndarray, ...]: + def compute( + self, *data: chunkedduckarray[Any, _DType], **kwargs + ) -> tuple[duckarray[Any, _DType], ...]: from dask.array import compute return compute(*data, **kwargs) From 9c619a21335012367cb813a1a18d68be5d9ee738 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:20:00 +0000 Subject: [PATCH 40/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_parallelcompat.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index c46ef1c2fdd..bf66b2ff6f8 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -6,13 +6,12 @@ import numpy as np import pytest -from xarray.core.types import T_DuckArray from xarray.namedarray._typing import ( _Chunks, - chunkedduckarray, _ChunksLike, - _Shape, _DType, + _Shape, + chunkedduckarray, duckarray, ) from xarray.namedarray.daskmanager import DaskManager From 36831ce9c7c1d299a54f209764b8bd0e1c5a176c Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:25:18 +0200 Subject: [PATCH 41/47] Update test_parallelcompat.py --- xarray/tests/test_parallelcompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index bf66b2ff6f8..1ff1414783b 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -85,7 +85,7 @@ def normalize_chunks( return normalize_chunks(chunks, shape, limit, dtype, previous_chunks) def from_array( - self, data: duckarray[Any, _DType], chunks: _Chunks, **kwargs + self, data: duckarray[Any, _DType], chunks: _ChunksLike, **kwargs ) -> chunkedduckarray[Any, _DType]: from dask import array as da From 0ec9be784ef4c6ae59501736916b9d5d9dd9619e Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:30:40 +0200 Subject: [PATCH 42/47] Update times.py --- xarray/coding/times.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index a5f18496661..a30affcbe93 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -706,6 +706,20 @@ def _cast_to_dtype_if_safe(num: np.ndarray, dtype: np.dtype) -> np.ndarray: return cast_num +@overload +def encode_cf_datetime( + dates: chunkedduckarray, + units: str | None = None, + calendar: str | None = None, + dtype: np.dtype | None = None, +) -> tuple[chunkedduckarray, str, str]: ... +@overload +def encode_cf_datetime( + dates: duckarray, + units: str | None = None, + calendar: str | None = None, + dtype: np.dtype | None = None, +) -> tuple[duckarray, str, str]: ... def encode_cf_datetime( dates: duckarray | chunkedduckarray, units: str | None = None, From 1e35da380a54c1c18837aa5ae5f0e155a53bc856 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:41:50 +0200 Subject: [PATCH 43/47] Update _typing.py --- xarray/namedarray/_typing.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/xarray/namedarray/_typing.py b/xarray/namedarray/_typing.py index b052e5ee76e..751fc92837a 100644 --- a/xarray/namedarray/_typing.py +++ b/xarray/namedarray/_typing.py @@ -157,9 +157,7 @@ def dtype(self) -> _DType_co: ... @runtime_checkable -class _arrayfunction( - _array[_ShapeType_co, _DType_co], Protocol[_ShapeType_co, _DType_co] -): +class _arrayfunction(_array[_ShapeType, _DType_co], Protocol[_ShapeType, _DType_co]): """ Duck array supporting NEP 18. @@ -185,14 +183,14 @@ def __getitem__( ) -> _arrayfunction[Any, _DType_co] | Any: ... @overload - def __array__(self, dtype: None = ..., /) -> np.ndarray[Any, _DType_co]: ... + def __array__(self, dtype: None = ..., /) -> np.ndarray[_ShapeType, _DType_co]: ... @overload - def __array__(self, dtype: _DType, /) -> np.ndarray[Any, _DType]: ... + def __array__(self, dtype: _DType, /) -> np.ndarray[_ShapeType, _DType]: ... def __array__( self, dtype: _DType | None = ..., / - ) -> np.ndarray[Any, _DType] | np.ndarray[Any, _DType_co]: ... + ) -> np.ndarray[_ShapeType, _DType] | np.ndarray[_ShapeType, _DType_co]: ... # TODO: Should return the same subclass but with a new dtype generic. # https://github.com/python/typing/issues/548 @@ -215,10 +213,10 @@ def __array_function__( ) -> Any: ... @property - def imag(self) -> _arrayfunction[_ShapeType_co, Any]: ... + def imag(self) -> _arrayfunction[_ShapeType, Any]: ... @property - def real(self) -> _arrayfunction[_ShapeType_co, Any]: ... + def real(self) -> _arrayfunction[_ShapeType, Any]: ... @runtime_checkable @@ -267,7 +265,7 @@ def chunks(self) -> _Chunks: ... @runtime_checkable class _chunkedarrayfunction( - _arrayfunction[_ShapeType_co, _DType_co], Protocol[_ShapeType_co, _DType_co] + _arrayfunction[_ShapeType, _DType_co], Protocol[_ShapeType, _DType_co] ): """ Chunked duck array supporting NEP 18. @@ -281,7 +279,7 @@ def chunks(self) -> _Chunks: ... def rechunk( self, chunks: _ChunksLike, - ) -> _chunkedarrayfunction[_ShapeType_co, _DType_co]: ... + ) -> _chunkedarrayfunction[_ShapeType, _DType_co]: ... @runtime_checkable @@ -326,7 +324,7 @@ def todense(self) -> np.ndarray[Any, np.dtype[np.generic]]: ... @runtime_checkable class _sparsearrayfunction( - _arrayfunction[_ShapeType_co, _DType_co], Protocol[_ShapeType_co, _DType_co] + _arrayfunction[_ShapeType, _DType_co], Protocol[_ShapeType, _DType_co] ): """ Sparse duck array supporting NEP 18. @@ -361,8 +359,18 @@ def todense(self) -> np.ndarray[Any, np.dtype[np.generic]]: ... ErrorOptionsWithWarn = Literal["raise", "warn", "ignore"] -def test(arr: duckarray[_ShapeType, _DType]) -> duckarray[_ShapeType, _DType]: - return arr +# def test(arr: duckarray[_ShapeType, _DType]) -> duckarray[_ShapeType, _DType]: +# return np.round(arr) + + +# test(np.array([], dtype=np.int64)) + + +# def test2(arr: _arrayfunction[Any, _DType]) -> _arrayfunction[Any, _DType]: +# return np.round(arr) +# # return np.asarray(arr) +# # return arr.__array__() +# # return arr -test(np.array([], dtype=np.int64)) +# test2(np.array([], dtype=np.int64)) From 6930c2232d8d0f72801d733069cf477d299d33ef Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Tue, 9 Jul 2024 23:14:49 +0200 Subject: [PATCH 44/47] Update test_coding_times.py --- xarray/tests/test_coding_times.py | 93 +++++++++++++++---------------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 393f8400c46..4e31c28b8d3 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -14,13 +14,14 @@ Dataset, Variable, cftime_range, - coding, conventions, date_range, decode_cf, ) from xarray.coding.times import ( + _STANDARD_CALENDARS, _encode_datetime_with_cftime, + _netcdf_to_numpy_timeunit, _numpy_to_netcdf_timeunit, _should_cftime_be_used, cftime_to_nptime, @@ -28,7 +29,11 @@ decode_cf_timedelta, encode_cf_datetime, encode_cf_timedelta, + format_cftime_datetime, + infer_datetime_units, + infer_timedelta_units, to_timedelta_unboxed, + CFDatetimeCoder, ) from xarray.coding.variables import SerializationWarning from xarray.conventions import _update_bounds_attributes, cf_encoder @@ -53,11 +58,9 @@ "all_leap", "366_day", } -_ALL_CALENDARS = sorted( - _NON_STANDARD_CALENDARS_SET.union(coding.times._STANDARD_CALENDARS) -) +_ALL_CALENDARS = sorted(_NON_STANDARD_CALENDARS_SET.union(_STANDARD_CALENDARS)) _NON_STANDARD_CALENDARS = sorted(_NON_STANDARD_CALENDARS_SET) -_STANDARD_CALENDARS = sorted(coding.times._STANDARD_CALENDARS) +_STANDARD_CALENDARS = sorted(_STANDARD_CALENDARS) _CF_DATETIME_NUM_DATES_UNITS = [ (np.arange(10), "days since 2000-01-01"), (np.arange(10).astype("float64"), "days since 2000-01-01"), @@ -130,7 +133,7 @@ def test_cf_datetime(num_dates, units, calendar) -> None: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - actual = coding.times.decode_cf_datetime(num_dates, units, calendar) + actual = decode_cf_datetime(num_dates, units, calendar) abs_diff = np.asarray(abs(actual - expected)).ravel() abs_diff = pd.to_timedelta(abs_diff.tolist()).to_numpy() @@ -139,17 +142,15 @@ def test_cf_datetime(num_dates, units, calendar) -> None: # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, "s")).all() - encoded, _, _ = coding.times.encode_cf_datetime(actual, units, calendar) + encoded, _, _ = encode_cf_datetime(actual, units, calendar) - assert_array_equal(num_dates, np.around(encoded, 1)) + assert_array_equal(num_dates, np.round(encoded, 1)) if hasattr(num_dates, "ndim") and num_dates.ndim == 1 and "1000" not in units: # verify that wrapping with a pandas.Index works # note that it *does not* currently work to put # non-datetime64 compatible dates into a pandas.Index - encoded, _, _ = coding.times.encode_cf_datetime( - pd.Index(actual), units, calendar - ) - assert_array_equal(num_dates, np.around(encoded, 1)) + encoded, _, _ = encode_cf_datetime(pd.Index(actual), units, calendar) + assert_array_equal(num_dates, np.round(encoded, 1)) @requires_cftime @@ -169,7 +170,7 @@ def test_decode_cf_datetime_overflow() -> None: for i, day in enumerate(days): with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - result = coding.times.decode_cf_datetime(day, units) + result = decode_cf_datetime(day, units) assert result == expected[i] @@ -178,7 +179,7 @@ def test_decode_cf_datetime_non_standard_units() -> None: # netCDFs from madis.noaa.gov use this format for their time units # they cannot be parsed by cftime, but pd.Timestamp works units = "hours since 1-1-1970" - actual = coding.times.decode_cf_datetime(np.arange(100), units) + actual = decode_cf_datetime(np.arange(100), units) assert_array_equal(actual, expected) @@ -193,7 +194,7 @@ def test_decode_cf_datetime_non_iso_strings() -> None: (np.arange(100), "hours since 2000-01-01 0:00"), ] for num_dates, units in cases: - actual = coding.times.decode_cf_datetime(num_dates, units) + actual = decode_cf_datetime(num_dates, units) abs_diff = abs(actual - expected.values) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: @@ -212,7 +213,7 @@ def test_decode_standard_calendar_inside_timestamp_range(calendar) -> None: expected = times.values expected_dtype = np.dtype("M8[ns]") - actual = coding.times.decode_cf_datetime(time, units, calendar=calendar) + actual = decode_cf_datetime(time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -235,9 +236,7 @@ def test_decode_non_standard_calendar_inside_timestamp_range(calendar) -> None: ) expected_dtype = np.dtype("O") - actual = coding.times.decode_cf_datetime( - non_standard_time, units, calendar=calendar - ) + actual = decode_cf_datetime(non_standard_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -264,7 +263,7 @@ def test_decode_dates_outside_timestamp_range(calendar) -> None: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - actual = coding.times.decode_cf_datetime(time, units, calendar=calendar) + actual = decode_cf_datetime(time, units, calendar=calendar) assert all(isinstance(value, expected_date_type) for value in actual) abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -282,7 +281,7 @@ def test_decode_standard_calendar_single_element_inside_timestamp_range( for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar) + actual = decode_cf_datetime(num_time, units, calendar=calendar) assert actual.dtype == np.dtype("M8[ns]") @@ -295,7 +294,7 @@ def test_decode_non_standard_calendar_single_element_inside_timestamp_range( for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar) + actual = decode_cf_datetime(num_time, units, calendar=calendar) assert actual.dtype == np.dtype("O") @@ -309,9 +308,7 @@ def test_decode_single_element_outside_timestamp_range(calendar) -> None: for num_time in [days, [days], [[days]]]: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar - ) + actual = decode_cf_datetime(num_time, units, calendar=calendar) expected = cftime.num2date( days, units, calendar, only_use_cftime_datetimes=True @@ -338,7 +335,7 @@ def test_decode_standard_calendar_multidim_time_inside_timestamp_range( expected1 = times1.values expected2 = times2.values - actual = coding.times.decode_cf_datetime(mdim_time, units, calendar=calendar) + actual = decode_cf_datetime(mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype("M8[ns]") abs_diff1 = abs(actual[:, 0] - expected1) @@ -379,7 +376,7 @@ def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( expected_dtype = np.dtype("O") - actual = coding.times.decode_cf_datetime(mdim_time, units, calendar=calendar) + actual = decode_cf_datetime(mdim_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff1 = abs(actual[:, 0] - expected1) @@ -412,7 +409,7 @@ def test_decode_multidim_time_outside_timestamp_range(calendar) -> None: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") - actual = coding.times.decode_cf_datetime(mdim_time, units, calendar=calendar) + actual = decode_cf_datetime(mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype("O") @@ -435,7 +432,7 @@ def test_decode_non_standard_calendar_single_element(calendar, num_time) -> None units = "days since 0001-01-01" - actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar) + actual = decode_cf_datetime(num_time, units, calendar=calendar) expected = np.asarray( cftime.num2date(num_time, units, calendar, only_use_cftime_datetimes=True) @@ -460,9 +457,7 @@ def test_decode_360_day_calendar() -> None: with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - actual = coding.times.decode_cf_datetime( - num_times, units, calendar=calendar - ) + actual = decode_cf_datetime(num_times, units, calendar=calendar) assert len(w) == 0 assert actual.dtype == np.dtype("O") @@ -476,8 +471,8 @@ def test_decode_abbreviation() -> None: val = np.array([1586628000000.0]) units = "msecs since 1970-01-01T00:00:00Z" - actual = coding.times.decode_cf_datetime(val, units) - expected = coding.times.cftime_to_nptime(cftime.num2date(val, units)) + actual = decode_cf_datetime(val, units) + expected = cftime_to_nptime(cftime.num2date(val, units)) assert_array_equal(actual, expected) @@ -498,7 +493,7 @@ def test_decode_abbreviation() -> None: def test_cf_datetime_nan(num_dates, units, expected_list) -> None: with warnings.catch_warnings(): warnings.filterwarnings("ignore", "All-NaN") - actual = coding.times.decode_cf_datetime(num_dates, units) + actual = decode_cf_datetime(num_dates, units) # use pandas because numpy will deprecate timezone-aware conversions expected = pd.to_datetime(expected_list).to_numpy(dtype="datetime64[ns]") assert_array_equal(expected, actual) @@ -510,7 +505,7 @@ def test_decoded_cf_datetime_array_2d() -> None: variable = Variable( ("x", "y"), np.array([[0, 1], [2, 3]]), {"units": "days since 2000-01-01"} ) - result = coding.times.CFDatetimeCoder().decode(variable) + result = CFDatetimeCoder().decode(variable) assert result.dtype == "datetime64[ns]" expected = pd.date_range("2000-01-01", periods=4).values.reshape(2, 2) assert_array_equal(np.asarray(result), expected) @@ -531,7 +526,7 @@ def test_decoded_cf_datetime_array_2d() -> None: def test_infer_datetime_units(freq, units) -> None: dates = pd.date_range("2000", periods=2, freq=freq) expected = f"{units} since 2000-01-01 00:00:00" - assert expected == coding.times.infer_datetime_units(dates) + assert expected == infer_datetime_units(dates) @pytest.mark.parametrize( @@ -549,7 +544,7 @@ def test_infer_datetime_units(freq, units) -> None: ], ) def test_infer_datetime_units_with_NaT(dates, expected) -> None: - assert expected == coding.times.infer_datetime_units(dates) + assert expected == infer_datetime_units(dates) _CFTIME_DATETIME_UNITS_TESTS = [ @@ -573,7 +568,7 @@ def test_infer_datetime_units_with_NaT(dates, expected) -> None: def test_infer_cftime_datetime_units(calendar, date_args, expected) -> None: date_type = _all_cftime_date_types()[calendar] dates = [date_type(*args) for args in date_args] - assert expected == coding.times.infer_datetime_units(dates) + assert expected == infer_datetime_units(dates) @pytest.mark.filterwarnings("ignore:Timedeltas can't be serialized faithfully") @@ -600,18 +595,18 @@ def test_cf_timedelta(timedeltas, units, numbers) -> None: numbers = np.array(numbers) expected = numbers - actual, _ = coding.times.encode_cf_timedelta(timedeltas, units) + actual, _ = encode_cf_timedelta(timedeltas, units) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype if units is not None: expected = timedeltas - actual = coding.times.decode_cf_timedelta(numbers, units) + actual = decode_cf_timedelta(numbers, units) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype expected = np.timedelta64("NaT", "ns") - actual = coding.times.decode_cf_timedelta(np.array(np.nan), "days") + actual = decode_cf_timedelta(np.array(np.nan), "days") assert_array_equal(expected, actual) @@ -622,7 +617,7 @@ def test_cf_timedelta_2d() -> None: timedeltas = np.atleast_2d(to_timedelta_unboxed(["1D", "2D", "3D"])) expected = timedeltas - actual = coding.times.decode_cf_timedelta(numbers, units) + actual = decode_cf_timedelta(numbers, units) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype @@ -637,7 +632,7 @@ def test_cf_timedelta_2d() -> None: ], ) def test_infer_timedelta_units(deltas, expected) -> None: - assert expected == coding.times.infer_timedelta_units(deltas) + assert expected == infer_timedelta_units(deltas) @requires_cftime @@ -653,7 +648,7 @@ def test_infer_timedelta_units(deltas, expected) -> None: def test_format_cftime_datetime(date_args, expected) -> None: date_types = _all_cftime_date_types() for date_type in date_types.values(): - result = coding.times.format_cftime_datetime(date_type(*date_args)) + result = format_cftime_datetime(date_type(*date_args)) assert result == expected @@ -1008,7 +1003,7 @@ def test_decode_ambiguous_time_warns(calendar) -> None: # we don't decode non-standard calendards with # pandas so expect no warning will be emitted - is_standard_calendar = calendar in coding.times._STANDARD_CALENDARS + is_standard_calendar = calendar in _STANDARD_CALENDARS dates = [1, 2, 3] units = "days since 1-1-1" @@ -1043,9 +1038,9 @@ def test_encode_cf_datetime_defaults_to_correct_dtype( pytest.skip("Nanosecond frequency is not valid for cftime dates.") times = date_range("2000", periods=3, freq=freq) units = f"{encoding_units} since 2000-01-01" - encoded, _units, _ = coding.times.encode_cf_datetime(times, units) + encoded, _units, _ = encode_cf_datetime(times, units) - numpy_timeunit = coding.times._netcdf_to_numpy_timeunit(encoding_units) + numpy_timeunit = _netcdf_to_numpy_timeunit(encoding_units) encoding_units_as_timedelta = np.timedelta64(1, numpy_timeunit) if pd.to_timedelta(1, freq) >= encoding_units_as_timedelta: assert encoded.dtype == np.int64 @@ -1202,7 +1197,7 @@ def test_decode_float_datetime(): def test_scalar_unit() -> None: # test that a scalar units (often NaN when using to_netcdf) does not raise an error variable = Variable(("x", "y"), np.array([[0, 1], [2, 3]]), {"units": np.nan}) - result = coding.times.CFDatetimeCoder().decode(variable) + result = CFDatetimeCoder().decode(variable) assert np.isnan(result.attrs["units"]) From 96dde9f91967bb618e349deb9e768fcec63ab232 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:17:59 +0000 Subject: [PATCH 45/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_coding_times.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 4e31c28b8d3..1114c98362c 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -20,6 +20,7 @@ ) from xarray.coding.times import ( _STANDARD_CALENDARS, + CFDatetimeCoder, _encode_datetime_with_cftime, _netcdf_to_numpy_timeunit, _numpy_to_netcdf_timeunit, @@ -33,7 +34,6 @@ infer_datetime_units, infer_timedelta_units, to_timedelta_unboxed, - CFDatetimeCoder, ) from xarray.coding.variables import SerializationWarning from xarray.conventions import _update_bounds_attributes, cf_encoder From 6f8a29c6ab1694368623196456232bd2e3eca57b Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:36:33 +0200 Subject: [PATCH 46/47] use duckarray assertions instead --- xarray/tests/test_coding_times.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index d568bdc3268..7633ee28a30 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -48,6 +48,8 @@ has_cftime, requires_cftime, requires_dask, + assert_duckarray_allclose, + assert_duckarray_equal, ) _NON_STANDARD_CALENDARS_SET = { @@ -144,13 +146,13 @@ def test_cf_datetime(num_dates, units, calendar) -> None: assert (abs_diff <= np.timedelta64(1, "s")).all() encoded, _, _ = encode_cf_datetime(actual, units, calendar) - assert_array_equal(num_dates, np.round(encoded, 1)) + assert_duckarray_allclose(num_dates, encoded) if hasattr(num_dates, "ndim") and num_dates.ndim == 1 and "1000" not in units: # verify that wrapping with a pandas.Index works # note that it *does not* currently work to put # non-datetime64 compatible dates into a pandas.Index encoded, _, _ = encode_cf_datetime(pd.Index(actual), units, calendar) - assert_array_equal(num_dates, np.round(encoded, 1)) + assert_duckarray_allclose(num_dates, encoded) @requires_cftime @@ -893,10 +895,10 @@ def test_time_units_with_timezone_roundtrip(calendar) -> None: ) if calendar in _STANDARD_CALENDARS: - np.testing.assert_array_equal(result_num_dates, expected_num_dates) + assert_duckarray_equal(result_num_dates, expected_num_dates) else: # cftime datetime arithmetic is not quite exact. - np.testing.assert_allclose(result_num_dates, expected_num_dates) + assert_duckarray_allclose(result_num_dates, expected_num_dates) assert result_units == expected_units assert result_calendar == calendar From f2903f8dd69b8ae849047855abc307ccb8eb467b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Jul 2024 09:37:15 +0000 Subject: [PATCH 47/47] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_coding_times.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 7633ee28a30..623e4e9f970 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -44,12 +44,12 @@ FirstElementAccessibleArray, arm_xfail, assert_array_equal, + assert_duckarray_allclose, + assert_duckarray_equal, assert_no_warnings, has_cftime, requires_cftime, requires_dask, - assert_duckarray_allclose, - assert_duckarray_equal, ) _NON_STANDARD_CALENDARS_SET = {