ENH: Allow SparseDataFrame/SparseSeries values assignment

Also fix .where for sparse blocks.
Discrepancy comes from:

	 dense_frame._data.blocks[0].values  # this is 2D even for 1D block
	sparse_frame._data.blocks[0].values  # this is always 1D

I'm sure this had worked before and was unneeded in Oct 2017.

Author: kernc

pandas/core/frame.py

@@ -2641,9 +2641,7 @@ def set_value(self, index, col, value, takeable=False):
 
         Returns
         -------
-        frame : DataFrame
-            If label pair is contained, will be reference to calling DataFrame,
-            otherwise a new object
+        self : DataFrame
         """
         warnings.warn("set_value is deprecated and will be removed "
                       "in a future release. Please use "

pandas/core/internals.py

@@ -924,6 +924,9 @@ def _is_empty_indexer(indexer):
         if _is_empty_indexer(indexer):
             pass
 
+        elif is_sparse(values):
+            values = values.set_values(indexer, value)
+
         # setting a single element for each dim and with a rhs that could
         # be say a list
         # GH 6043
@@ -1497,6 +1500,11 @@ def where(self, other, cond, align=True, errors='raise',
             raise ValueError("where must have a condition that is ndarray "
                              "like")
 
+        # For SparseBlock, self.values is always 1D. If cond was a frame,
+        # it's 2D values would incorrectly broadcast later on.
+        if values.ndim == 1 and any(ax == 1 for ax in cond.shape):
+            cond = cond.ravel()
+
         # our where function
         def func(cond, values, other):
             if cond.ravel().all():
@@ -1808,6 +1816,11 @@ def putmask(self, mask, new, align=True, inplace=False, axis=0,
         new_values = self.values if inplace else self.copy().values
         new_values, _, new, _ = self._try_coerce_args(new_values, new)
 
+        if is_sparse(new_values):
+            indexer = mask.to_dense().values.ravel().nonzero()[0]
+            block = self.setitem(indexer, new)
+            return [block]
+
         if isinstance(new, np.ndarray) and len(new) == len(mask):
             new = new[mask]
 
@@ -3060,6 +3073,17 @@ def _astype(self, dtype, copy=False, errors='raise', values=None,
         return self.make_block_same_class(values=values,
                                           placement=self.mgr_locs)
 
+    def _can_hold_element(self, element):
+        return np.can_cast(np.asarray(element).dtype, self.sp_values.dtype)
+
+    def _try_coerce_result(self, result):
+        if (isinstance(result, np.ndarray) and
+                np.ndim(result) == 1 and
+                not is_sparse(result)):
+            result = SparseArray(result, kind=self.kind,
+                                 fill_value=self.fill_value)
+        return result
+
     def __len__(self):
         try:
             return self.sp_index.length

pandas/core/series.py

@@ -1067,9 +1067,7 @@ def set_value(self, label, value, takeable=False):
 
         Returns
         -------
-        series : Series
-            If label is contained, will be reference to calling Series,
-            otherwise a new object
+        self : Series
         """
         warnings.warn("set_value is deprecated and will be removed "
                       "in a future release. Please use "

pandas/core/sparse/array.py

@@ -37,6 +37,7 @@
 import pandas.core.algorithms as algos
 import pandas.core.ops as ops
 import pandas.io.formats.printing as printing
+from pandas.errors import PerformanceWarning
 from pandas.util._decorators import Appender
 from pandas.core.indexes.base import _index_shared_docs
 
@@ -369,6 +370,53 @@ def get_values(self, fill=None):
         """ return a dense representation """
         return self.to_dense(fill=fill)
 
+    def set_values(self, indexer, value):
+        """
+        Return new SparseArray with indexed values set to `value`.
+
+        Returns
+        -------
+        SparseArray
+            A new sparse array with indexer positions filled with value.
+        """
+        # If indexer is not a single int position, easiest to handle via dense
+        if not is_scalar(indexer):
+            warnings.warn(
+                'Setting SparseSeries/Array values is particularly '
+                'inefficient when indexing with multiple keys because the '
+                'whole series is made dense interim.',
+                PerformanceWarning, stacklevel=2)
+
+            values = self.to_dense()
+            values[indexer] = value
+            return SparseArray(values, kind=self.kind,
+                               fill_value=self.fill_value)
+
+        warnings.warn(
+            'Setting SparseSeries/Array values is inefficient '
+            '(a copy of data is made).', PerformanceWarning, stacklevel=2)
+
+        # If label already in sparse index, just switch the value on a copy
+        idx = self.sp_index.lookup(indexer)
+        if idx != -1:
+            obj = self.copy()
+            obj.sp_values[idx] = value
+            return obj
+
+        # Otherwise, construct a new array, and insert the new value in the
+        # correct position
+        indices = self.sp_index.to_int_index().indices
+        pos = np.searchsorted(indices, indexer)
+
+        indices = np.insert(indices, pos, indexer)
+        sp_values = np.insert(self.sp_values, pos, value)
+        # Length can be increased when adding a new value into index
+        length = max(self.sp_index.length, indexer + 1)
+        sp_index = _make_index(length, indices, self.kind)
+
+        return SparseArray(sp_values, sparse_index=sp_index,
+                           fill_value=self.fill_value)
+
     def to_dense(self, fill=None):
         """
         Convert SparseArray to a NumPy array.
@@ -544,6 +592,10 @@ def astype(self, dtype=None, copy=True):
         return self._simple_new(sp_values, self.sp_index,
                                 fill_value=fill_value)
 
+    def tolist(self):
+        """Return *dense* self as list"""
+        return self.values.tolist()
+
     def copy(self, deep=True):
         """
         Make a copy of the SparseArray. Only the actual sparse values need to

pandas/core/sparse/frame.py

@@ -332,8 +332,8 @@ def _apply_columns(self, func):
             default_fill_value=self.default_fill_value,
             default_kind=self.default_kind).__finalize__(self)
 
-    def astype(self, dtype):
-        return self._apply_columns(lambda x: x.astype(dtype))
+    def astype(self, dtype, **kwargs):
+        return self._apply_columns(lambda x: x.astype(dtype, **kwargs))
 
     def copy(self, deep=True):
         """
@@ -464,44 +464,6 @@ def _get_value(self, index, col, takeable=False):
         return series._get_value(index, takeable=takeable)
     _get_value.__doc__ = get_value.__doc__
 
-    def set_value(self, index, col, value, takeable=False):
-        """
-        Put single value at passed column and index
-
-        .. deprecated:: 0.21.0
-
-        Please use .at[] or .iat[] accessors.
-
-        Parameters
-        ----------
-        index : row label
-        col : column label
-        value : scalar value
-        takeable : interpret the index/col as indexers, default False
-
-        Notes
-        -----
-        This method *always* returns a new object. It is currently not
-        particularly efficient (and potentially very expensive) but is provided
-        for API compatibility with DataFrame
-
-        Returns
-        -------
-        frame : DataFrame
-        """
-        warnings.warn("set_value is deprecated and will be removed "
-                      "in a future release. Please use "
-                      ".at[] or .iat[] accessors instead", FutureWarning,
-                      stacklevel=2)
-        return self._set_value(index, col, value, takeable=takeable)
-
-    def _set_value(self, index, col, value, takeable=False):
-        dense = self.to_dense()._set_value(
-            index, col, value, takeable=takeable)
-        return dense.to_sparse(kind=self._default_kind,
-                               fill_value=self._default_fill_value)
-    _set_value.__doc__ = set_value.__doc__
-
     def _slice(self, slobj, axis=0, kind=None):
         if axis == 0:
             new_index = self.index[slobj]

pandas/core/sparse/series.py

@@ -9,6 +9,7 @@
 import warnings
 
 from pandas.core.dtypes.missing import isna, notna
+from pandas.core.dtypes.common import is_sparse
 
 from pandas.compat.numpy import function as nv
 from pandas.core.index import Index, _ensure_index, InvalidIndexError
@@ -17,7 +18,6 @@
 from pandas.core import generic
 import pandas.core.common as com
 import pandas.core.ops as ops
-import pandas._libs.index as libindex
 from pandas.util._decorators import Appender
 
 from pandas.core.sparse.array import (
@@ -277,8 +277,13 @@ def __array_wrap__(self, result, context=None):
         else:
             fill_value = self.fill_value
 
+        # Assume: If result size matches, old sparse index is valid (ok???)
+        if np.size(result) == self.sp_index.npoints:
+            sp_index = self.sp_index
+        else:
+            sp_index = None
         return self._constructor(result, index=self.index,
-                                 sparse_index=self.sp_index,
+                                 sparse_index=sp_index,
                                  fill_value=fill_value,
                                  copy=False).__finalize__(self)
 
@@ -479,7 +484,7 @@ def set_value(self, label, value, takeable=False):
 
         Returns
         -------
-        series : SparseSeries
+        self : SparseSeries
         """
         warnings.warn("set_value is deprecated and will be removed "
                       "in a future release. Please use "
@@ -488,35 +493,16 @@ def set_value(self, label, value, takeable=False):
         return self._set_value(label, value, takeable=takeable)
 
     def _set_value(self, label, value, takeable=False):
-        values = self.to_dense()
-
-        # if the label doesn't exist, we will create a new object here
-        # and possibly change the index
-        new_values = values._set_value(label, value, takeable=takeable)
-        if new_values is not None:
-            values = new_values
-        new_index = values.index
-        values = SparseArray(values, fill_value=self.fill_value,
-                             kind=self.kind)
-        self._data = SingleBlockManager(values, new_index)
-        self._index = new_index
+        self._data = self._data.copy()
+        try:
+            idx = self.index.get_loc(label)
+        except KeyError:
+            idx = len(self)
+            self._data.axes[0] = self._data.index.append(Index([label]))
+        self._data = self._data.setitem(indexer=idx, value=value)
+        return self
     _set_value.__doc__ = set_value.__doc__
 
-    def _set_values(self, key, value):
-
-        # this might be inefficient as we have to recreate the sparse array
-        # rather than setting individual elements, but have to convert
-        # the passed slice/boolean that's in dense space into a sparse indexer
-        # not sure how to do that!
-        if isinstance(key, Series):
-            key = key.values
-
-        values = self.values.to_dense()
-        values[key] = libindex.convert_scalar(values, value)
-        values = SparseArray(values, fill_value=self.fill_value,
-                             kind=self.kind)
-        self._data = SingleBlockManager(values, self.index)
-
     def to_dense(self, sparse_only=False):
         """
         Convert SparseSeries to a Series.
@@ -542,8 +528,10 @@ def to_dense(self, sparse_only=False):
             index = self.index.take(int_index.indices)
             return Series(self.sp_values, index=index, name=self.name)
         else:
-            return Series(self.values.to_dense(), index=self.index,
-                          name=self.name)
+            values = self.values
+            if is_sparse(values):
+                values = values.to_dense()
+            return Series(values, index=self.index, name=self.name)
 
     @property
     def density(self):