Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dask: Data.varray #397

Merged
merged 5 commits into from
May 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 3 additions & 17 deletions cf/data/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -845,7 +845,7 @@ def __bool__(self):
"elements is ambiguous. Use d.any() or d.all()"
)

return bool(self._get_dask())
return bool(self.to_dask_array())

def __repr__(self):
"""Called by the `repr` built-in function.
Expand Down Expand Up @@ -5199,20 +5199,6 @@ def datetime_array(self):

return a

@property
@daskified(_DASKIFIED_VERBOSE)
def varray(self):
"""A numpy array view of the data array.

Deprecated at version TODODASK.

.. seealso:: `array`, `datetime_array`, `compute`, `persist`

"""
raise NotImplementedError(
"The varray method was deprecated at version TODODASK"
)

@property
@daskified(_DASKIFIED_VERBOSE)
def mask(self):
Expand Down Expand Up @@ -5665,7 +5651,7 @@ def all(self, axis=None, keepdims=True, split_every=None):

"""
d = self.copy(array=False)
dx = self._get_dask()
dx = self.to_dask_array()
sadielbartholomew marked this conversation as resolved.
Show resolved Hide resolved
dx = da.all(dx, axis=axis, keepdims=keepdims, split_every=split_every)
d._set_dask(dx, reset_mask_hardness=False)
d.hardmask = _DEFAULT_HARDMASK
Expand Down Expand Up @@ -5782,7 +5768,7 @@ def any(self, axis=None, keepdims=True, split_every=None):

"""
d = self.copy(array=False)
dx = self._get_dask()
dx = self.to_dask_array()
dx = da.any(dx, axis=axis, keepdims=keepdims, split_every=split_every)
d._set_dask(dx, reset_mask_hardness=False)
d.hardmask = _DEFAULT_HARDMASK
Expand Down
185 changes: 109 additions & 76 deletions cf/data/mixin/deprecations.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,7 @@ def __hash__(self):
_DEPRECATION_ERROR_METHOD(
self,
"__hash__",
message="Consider using 'cf.hash_array' on the underlying "
"array instead.",
message="Consider using 'cf.hash_array' function array instead.",
version="TODODASK",
removed_at="5.0.0",
)
Expand Down Expand Up @@ -99,13 +98,22 @@ def _HDF_chunks(self):
def Data(self):
"""Deprecated at version 3.0.0, use attribute `data` instead."""
_DEPRECATION_ERROR_ATTRIBUTE(
self, "Data", "Use attribute 'data' instead."
self,
"Data",
"Use attribute 'data' instead.",
version="3.0.0",
removed_at="4.0.0",
) # pragma: no cover

@property
def dtvarray(self):
"""Deprecated at version 3.0.0."""
_DEPRECATION_ERROR_ATTRIBUTE(self, "dtvarray") # pragma: no cover
_DEPRECATION_ERROR_ATTRIBUTE(
self,
"dtvarray",
version="3.0.0",
removed_at="4.0.0",
) # pragma: no cover

@property
def in_memory(self):
Expand All @@ -121,22 +129,99 @@ def in_memory(self):
removed_at="5.0.0",
) # pragma: no cover

def files(self):
"""Deprecated at version 3.4.0, use method `get_filenames`
instead."""
_DEPRECATION_ERROR_METHOD(
@property
def ismasked(self):
"""True if the data array has any masked values.

Deprecated at version TODODASK. Use the `is_masked` attribute
instead.

**Examples**

>>> d = cf.Data([[1, 2, 3], [4, 5, 6]])
>>> print(d.ismasked)
False
>>> d[0, ...] = cf.masked
>>> d.ismasked
True

"""
_DEPRECATION_ERROR_ATTRIBUTE(
self,
"files",
"Use method `get_filenames` instead.",
version="3.4.0",
"ismasked",
message="Use the 'is_masked' attribute instead",
version="TODODASK",
removed_at="5.0.0",
) # pragma: no cover

@property
def ispartitioned(self):
"""True if the data array is partitioned.

**Examples**

>>> d._pmsize
1
>>> d.ispartitioned
False

>>> d._pmsize
2
>>> d.ispartitioned
False

"""
_DEPRECATION_ERROR_ATTRIBUTE(
self, "ispartitioned", version="TODODASK", removed_at="5.0.0"
) # pragma: no cover

@property
def unsafe_array(self):
"""Deprecated at version 3.0.0, use `array` attribute
instead."""
"""Deprecated at version 3.0.0.

Use the `array` attribute instead.

"""
_DEPRECATION_ERROR_ATTRIBUTE(
self,
"unsafe_array",
message="Use the 'array' attribute instead.",
version="3.0.0",
removed_at="4.0.0",
) # pragma: no cover

@property
def varray(self):
"""A numpy array view of the data array.

Deprecated at version TODODASK. Data are now stored as `dask`
arrays for which, in general, a numpy array view is not
robust.

Note that making changes to elements of the returned view changes
the underlying data.

.. seealso:: `array`, `to_dask_array`, `datetime_array`

**Examples**

>>> a = d.varray
>>> type(a)
<type 'numpy.ndarray'>
>>> a
array([0, 1, 2, 3, 4])
>>> a[0] = 999
>>> d.varray
array([999, 1, 2, 3, 4])

"""
_DEPRECATION_ERROR_ATTRIBUTE(
self, "unsafe_array", "Use 'array' attribute instead."
self,
"varray",
message="Data are now stored as `dask` arrays for which, "
"in general, a numpy array view is not robust.",
version="TODODASK",
removed_at="5.0.0",
) # pragma: no cover

def expand_dims(self, position=0, i=False):
Expand All @@ -149,6 +234,16 @@ def expand_dims(self, position=0, i=False):
version="3.0.0",
) # pragma: no cover

def files(self):
"""Deprecated at version 3.4.0, use method `get_filenames` instead."""
_DEPRECATION_ERROR_METHOD(
self,
"files",
"Use method `get_filenames` instead.",
version="3.4.0",
removed_at="4.0.0",
) # pragma: no cover

def fits_in_one_chunk_in_memory(self, itemsize):
"""Return True if the master array is small enough to be
retained in memory.
Expand Down Expand Up @@ -177,25 +272,6 @@ def fits_in_one_chunk_in_memory(self, itemsize):
removed_at="5.0.0",
) # pragma: no cover

@property
def ispartitioned(self):
"""True if the data array is partitioned.

**Examples**

>>> d._pmsize
1
>>> d.ispartitioned
False

>>> d._pmsize
2
>>> d.ispartitioned
False

"""
_DEPRECATION_ERROR_METHOD("TODODASK") # pragma: no cover

def close(self):
"""Close all files referenced by the data array.

Expand Down Expand Up @@ -481,49 +557,6 @@ def loads(self, j, chunk=True):
removed_at="5.0.0",
) # pragma: no cover

@property
def ismasked(self):
"""True if the data array has any masked values.

TODODASK

**Examples**

>>> d = cf.Data([[1, 2, 3], [4, 5, 6]])
>>> print(d.ismasked)
False
>>> d[0, ...] = cf.masked
>>> d.ismasked
True

"""
_DEPRECATION_ERROR_METHOD(
"TODODASK use is_masked instead"
) # pragma: no cover

@property
def varray(self):
"""A numpy array view the data array.

Note that making changes to elements of the returned view changes
the underlying data.

.. seealso:: `array`, `datetime_array`

**Examples**

>>> a = d.varray
>>> type(a)
<type 'numpy.ndarray'>
>>> a
array([0, 1, 2, 3, 4])
>>> a[0] = 999
>>> d.varray
array([999, 1, 2, 3, 4])

"""
_DEPRECATION_ERROR_METHOD("TODODASK") # pragma: no cover

def add_partitions(self, extra_boundaries, pdim):
"""Add partition boundaries.

Expand Down