Groupby sort

doc/source/api.rst

@@ -620,6 +620,14 @@ Serialization / IO / Conversion
    Series.to_string
    Series.to_clipboard
 
+Sparse methods
+~~~~~~~~~~~~~~
+.. autosummary::
+   :toctree: generated/
+
+   SparseSeries.to_coo
+   SparseSeries.from_coo
+
 .. _api.dataframe:
 
 DataFrame

doc/source/sparse.rst

@@ -109,10 +109,9 @@ accept scalar values or any 1-dimensional sequence:
 .. ipython:: python
    :suppress:
 
-   from numpy import nan
-
 .. ipython:: python
 
+   from numpy import nan
    spl.append(np.array([1., nan, nan, 2., 3.]))
    spl.append(5)
    spl.append(sparr)
@@ -135,3 +134,90 @@ recommend using ``block`` as it's more memory efficient. The ``integer`` format
 keeps an arrays of all of the locations where the data are not equal to the
 fill value. The ``block`` format tracks only the locations and sizes of blocks
 of data.
+
+.. _sparse.scipysparse:
+
+Interaction with scipy.sparse
+-----------------------------
+
+Experimental api to transform between sparse pandas and scipy.sparse structures.
+
+A :meth:`SparseSeries.to_coo` method is implemented for transforming a ``SparseSeries`` indexed by a ``MultiIndex`` to a ``scipy.sparse.coo_matrix``.
+
+The method requires a ``MultiIndex`` with two or more levels.
+
+.. ipython:: python
+   :suppress:
+
+
+.. ipython:: python
+
+   from numpy import nan
+   s = Series([3.0, nan, 1.0, 3.0, nan, nan])
+   s.index = MultiIndex.from_tuples([(1, 2, 'a', 0),
+                                     (1, 2, 'a', 1),
+                                     (1, 1, 'b', 0),
+                                     (1, 1, 'b', 1),
+                                     (2, 1, 'b', 0),
+                                     (2, 1, 'b', 1)],
+                                     names=['A', 'B', 'C', 'D'])
+
+   s
+   # SparseSeries
+   ss = s.to_sparse()
+   ss
+
+In the example below, we transform the ``SparseSeries`` to a sparse representation of a 2-d array by specifying that the first and second ``MultiIndex`` levels define labels for the rows and the third and fourth levels define labels for the columns. We also specify that the column and row labels should be sorted in the final sparse representation.
+
+.. ipython:: python
+
+   A, il, jl = ss.to_coo(ilevels=['A', 'B'], jlevels=['C', 'D'],
+                         sort_labels=True)
+
+   A
+   A.todense()
+   il
+   jl
+
+Specifying different row and column labels (and not sorting them) yields a different sparse matrix:
+
+.. ipython:: python
+
+   A, il, jl = ss.to_coo(ilevels=['A', 'B', 'C'], jlevels=['D'],
+                         sort_labels=False)
+
+   A
+   A.todense()
+   il
+   jl
+
+A convenience method :meth:`SparseSeries.from_coo` is implemented for creating a ``SparseSeries`` from a ``scipy.sparse.coo_matrix``.
+
+.. ipython:: python
+   :suppress:
+
+.. ipython:: python
+
+   from scipy import sparse
+   A = sparse.coo_matrix(([3.0, 1.0, 2.0], ([1, 0, 0], [0, 2, 3])),
+                               shape=(3, 4))
+   A
+   A.todense()
+
+The default behaviour (with ``dense_index=False``) simply returns a ``SparseSeries`` containing
+only the non-null entries.
+
+.. ipython:: python
+
+   ss = SparseSeries.from_coo(A)
+   ss
+
+Specifying ``dense_index=True`` will result in an index that is the Cartesian product of the
+row and columns coordinates of the matrix. Note that this will consume a significant amount of memory
+(relative to ``dense_index=False``) if the sparse matrix is large (and sparse) enough.
+
+.. ipython:: python
+
+   ss_dense = SparseSeries.from_coo(A, dense_index=True)
+   ss_dense
+

doc/source/whatsnew/v0.16.0.txt

@@ -110,6 +110,54 @@ Enhancements
 
 - Added auto-complete for ``Series.str.<tab>``, ``Series.dt.<tab>`` and ``Series.cat.<tab>`` (:issue:`9322`)
 
+Interaction with scipy.sparse
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- Added :meth:`SparseSeries.to_coo` and :meth:`SparseSeries.from_coo` methods
+  (:issue:`8048`) for converting to and from ``scipy.sparse.coo_matrix``
+  instances (see :ref:`here <sparse.scipysparse>`).
+  For example, given a SparseSeries with MultiIndex we can convert to a
+  `scipy.sparse.coo_matrix` by specifying the row and column labels as
+  index levels:
+
+  .. ipython:: python
+
+     from numpy import nan
+     s = Series([3.0, nan, 1.0, 3.0, nan, nan])
+     s.index = MultiIndex.from_tuples([(1, 2, 'a', 0),
+                                       (1, 2, 'a', 1),
+                                       (1, 1, 'b', 0),
+                                       (1, 1, 'b', 1),
+                                       (2, 1, 'b', 0),
+                                       (2, 1, 'b', 1)],
+                                       names=['A', 'B', 'C', 'D'])
+
+     s
+     # SparseSeries
+     ss = s.to_sparse()
+     ss
+
+     A, il, jl = ss.to_coo(ilevels=['A', 'B'], jlevels=['C', 'D'],
+                           sort_labels=False)
+
+     A
+     A.todense()
+     il
+     jl
+
+  The from_coo method is a convenience method for creating a ``SparseSeries``
+  from a ``scipy.sparse.coo_matrix``:
+
+  .. ipython:: python
+
+     from scipy import sparse
+     A = sparse.coo_matrix(([3.0, 1.0, 2.0], ([1, 0, 0], [0, 2, 3])),
+                                 shape=(3, 4))
+     A
+     A.todense()
+
+     ss = SparseSeries.from_coo(A)
+     ss
+
 Performance
 ~~~~~~~~~~~
 
@@ -210,3 +258,4 @@ Bug Fixes
 - Fixes issue with ``index_col=False`` when ``usecols`` is also specified in ``read_csv``. (:issue:`9082`)
 - Bug where ``wide_to_long`` would modify the input stubnames list (:issue:`9204`)
 - Bug in to_sql not storing float64 values using double precision. (:issue:`9009`)
+- Fixed bug in ``Series.groupby`` where grouping on ``MultiIndex`` levels would ignore the sort argument (:issue:`9444`)

pandas/core/groupby.py

@@ -1378,7 +1378,7 @@ def _get_compressed_labels(self):
         else:
             if len(all_labels) > 1:
                 group_index = get_group_index(all_labels, self.shape)
-                comp_ids, obs_group_ids = _compress_group_index(group_index)
+                comp_ids, obs_group_ids = _compress_group_index(group_index, sort=self.sort)
             else:
                 ping = self.groupings[0]
                 comp_ids = ping.labels

pandas/sparse/scipy_sparse.py

@@ -0,0 +1,103 @@
+"""
+Interaction with scipy.sparse matrices.
+
+Currently only includes SparseSeries.to_coo helpers.
+"""
+from pandas.core.frame import DataFrame
+from pandas.core.index import MultiIndex, Index
+from pandas.core.series import Series
+import itertools
+import numpy
+from pandas.compat import OrderedDict
+from pandas.tools.util import cartesian_product
+
+
+def _get_label_to_i_dict(labels, sort_labels=False):
+    """ Return OrderedDict of unique labels to number. Optionally sort by label. """
+    labels = Index(map(tuple, labels)).unique().tolist()  # squish
+    if sort_labels:
+        labels = sorted(list(labels))
+    d = OrderedDict((k, i) for i, k in enumerate(labels))
+    return(d)
+
+
+def _get_index_subset_to_coord_dict(index, subset, sort_labels=False):
+    ilabels = list(zip(*[index.get_level_values(i) for i in subset]))
+    labels_to_i = _get_label_to_i_dict(ilabels, sort_labels=sort_labels)
+    return(labels_to_i)
+
+
+def _check_is_partition(parts, whole):
+    whole = set(whole)
+    parts = [set(x) for x in parts]
+    if set.intersection(*parts) != set():
+        raise ValueError(
+            'Is not a partition because intersection is not null.')
+    if set.union(*parts) != whole:
+        raise ValueError('Is not a partition becuase union is not the whole.')
+
+
+def _to_ijv(ss, ilevels=(0,), jlevels=(1,), sort_labels=False):
+    """ For arbitrary (MultiIndexed) SparseSeries return
+    (v, i, j, ilabels, jlabels) where (v, (i, j)) is suitable for
+    passing to scipy.sparse.coo constructor. """
+    # index and column levels must be a partition of the index
+    _check_is_partition([ilevels, jlevels], range(ss.index.nlevels))
+
+    # from the SparseSeries: get the labels and data for non-null entries
+    values = ss._data.values._valid_sp_values
+    blocs = ss._data.values.sp_index.blocs
+    blength = ss._data.values.sp_index.blengths
+    nonnull_labels = list(
+        itertools.chain(*[ss.index.values[i:(i + j)] for i, j in zip(blocs, blength)]))
+
+    def get_indexers(levels):
+        """ Return sparse coords and dense labels for subset levels """
+        values_ilabels = [tuple(x[i] for i in levels) for x in nonnull_labels]
+        labels_to_i = _get_index_subset_to_coord_dict(
+            ss.index, levels, sort_labels=sort_labels)
+        i_coord = [labels_to_i[i] for i in values_ilabels]
+        return(i_coord, list(labels_to_i.keys()))
+
+    i_coord, i_labels = get_indexers(ilevels)
+    j_coord, j_labels = get_indexers(jlevels)
+
+    return(values, i_coord, j_coord, i_labels, j_labels)
+
+
+def _sparse_series_to_coo(ss, ilevels=(0,), jlevels=(1,), sort_labels=False):
+    """ Convert a SparseSeries to a scipy.sparse.coo_matrix using index levels ilevels, jlevels as the row and column
+    labels respectively. Returns the sparse_matrix, row and column labels. """
+    if ss.index.nlevels < 2:
+        raise ValueError('to_coo requires MultiIndex with nlevels > 2')
+    if not ss.index.is_unique:
+        raise ValueError(
+            'Duplicate index entries are not allowed in to_coo transformation.')
+
+    # to keep things simple, only rely on integer indexing (not labels)
+    ilevels = [ss.index._get_level_number(x) for x in ilevels]
+    jlevels = [ss.index._get_level_number(x) for x in jlevels]
+    ss = ss.copy()
+    ss.index.names = [None] * ss.index.nlevels  # kill any existing labels
+
+    v, i, j, il, jl = _to_ijv(
+        ss, ilevels=ilevels, jlevels=jlevels, sort_labels=sort_labels)
+    import scipy.sparse
+    sparse_matrix = scipy.sparse.coo_matrix(
+        (v, (i, j)), shape=(len(il), len(jl)))
+    return(sparse_matrix, il, jl)
+
+
+def _coo_to_sparse_series(A, dense_index=False):
+    """ Convert a scipy.sparse.coo_matrix to a SparseSeries.
+    Use the defaults given in the SparseSeries constructor. """
+    s = Series(A.data, MultiIndex.from_arrays((A.row, A.col)))
+    s = s.sort_index()
+    s = s.to_sparse()  # TODO: specify kind?
+    if dense_index:
+        # is there a better constructor method to use here?
+        i = range(A.shape[0])
+        j = range(A.shape[1])
+        ind = MultiIndex.from_product([i, j])
+        s = s.reindex_axis(ind)
+    return(s)

pandas/sparse/series.py

@@ -29,12 +29,14 @@
 
 from pandas.util.decorators import Appender
 
+from pandas.sparse.scipy_sparse import _sparse_series_to_coo, _coo_to_sparse_series
+
 #------------------------------------------------------------------------------
 # Wrapper function for Series arithmetic methods
 
 
 def _arith_method(op, name, str_rep=None, default_axis=None, fill_zeros=None,
-                                 **eval_kwargs):
+                  **eval_kwargs):
     """
     Wrapper function for Series arithmetic operations, to avoid
     code duplication.
@@ -654,6 +656,49 @@ def combine_first(self, other):
         dense_combined = self.to_dense().combine_first(other)
         return dense_combined.to_sparse(fill_value=self.fill_value)
 
+    def to_coo(self, ilevels=(0,), jlevels=(1,), sort_labels=False):
+        """
+        Create a scipy.sparse.coo_matrix from a SparseSeries with MultiIndex.
+
+        Use ilevels and jlevels to determine the row and column coordinates respectively.
+        ilevels and jlevels are the names (labels) or numbers of the levels.
+        {ilevels, jlevels} must be a partition of the MultiIndex level names (or numbers).
+
+        Parameters
+        ----------
+        ilevels : tuple/list
+        jlevels : tuple/list
+        sort_labels : bool, default False
+            Sort the row and column labels before forming the sparse matrix.
+
+        Returns
+        -------
+        y : scipy.sparse.coo_matrix
+        il : list (row labels)
+        jl : list (column labels)
+        """
+        A, il, jl = _sparse_series_to_coo(
+            self, ilevels, jlevels, sort_labels=sort_labels)
+        return(A, il, jl)
+
+    @classmethod
+    def from_coo(cls, A, dense_index=False):
+        """
+        Create a SparseSeries from a scipy.sparse.coo_matrix.
+
+        Parameters
+        ----------
+        A : scipy.sparse.coo_matrix
+        dense_index : bool, default False
+            If False (default), the SparseSeries index consists of only the coords of the non-null entries of the original coo_matrix.
+            If True, the SparseSeries index consists of the full sorted (row, col) coordinates of the coo_matrix.
+
+        Returns
+        -------
+        s : SparseSeries
+        """
+        return(_coo_to_sparse_series(A, dense_index=dense_index))
+
 # overwrite series methods with unaccelerated versions
 ops.add_special_arithmetic_methods(SparseSeries, use_numexpr=False,
                                    **ops.series_special_funcs)