Merge remote-tracking branch 'upstream/master' into fix/dask-computes

* upstream/master:
  Remove deprecated behavior from dataset.drop docstring (pydata#3451)
  jupyterlab dark theme (pydata#3443)
  Drop groups associated with nans in group variable (pydata#3406)
  Allow ellipsis (...) in transpose (pydata#3421)
  Another groupby.reduce bugfix. (pydata#3403)
  add icomoon license (pydata#3448)

README.rst

@@ -138,4 +138,7 @@ under a "3-clause BSD" license:
 xarray also bundles portions of CPython, which is available under the "Python
 Software Foundation License" in xarray/core/pycompat.py.
 
+xarray uses icons from the icomoon package (free version), which is
+available under the "CC BY 4.0" license.
+
 The full text of these licenses are included in the licenses directory.

doc/reshaping.rst

@@ -18,12 +18,14 @@ Reordering dimensions
 ---------------------
 
 To reorder dimensions on a :py:class:`~xarray.DataArray` or across all variables
-on a :py:class:`~xarray.Dataset`, use :py:meth:`~xarray.DataArray.transpose`:
+on a :py:class:`~xarray.Dataset`, use :py:meth:`~xarray.DataArray.transpose`. An 
+ellipsis (`...`) can be use to represent all other dimensions:
 
 .. ipython:: python
 
     ds = xr.Dataset({'foo': (('x', 'y', 'z'), [[[42]]]), 'bar': (('y', 'z'), [[24]])})
     ds.transpose('y', 'z', 'x')
+    ds.transpose(..., 'x')  # equivalent
     ds.transpose()  # reverses all dimensions
 
 Expand and squeeze dimensions

doc/whats-new.rst

@@ -25,6 +25,10 @@ Breaking changes
 
 New Features
 ~~~~~~~~~~~~
+- :py:meth:`Dataset.transpose` and :py:meth:`DataArray.transpose` now support an ellipsis (`...`)
+  to represent all 'other' dimensions. For example, to move one dimension to the front,
+  use `.transpose('x', ...)`. (:pull:`3421`)
+  By `Maximilian Roos <https://github.com/max-sixty>`_
 - Changed `xr.ALL_DIMS` to equal python's `Ellipsis` (`...`), and changed internal usages to use
   `...` directly. As before, you can use this to instruct a `groupby` operation
   to reduce over all dimensions. While we have no plans to remove `xr.ALL_DIMS`, we suggest
@@ -51,10 +55,13 @@ Bug fixes
 ~~~~~~~~~
 - Fix regression introduced in v0.14.0 that would cause a crash if dask is installed
   but cloudpickle isn't (:issue:`3401`) by `Rhys Doyle <https://github.com/rdoyle45>`_
-
-- Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4. 
+- Fix grouping over variables with NaNs. (:issue:`2383`, :pull:`3406`).
+  By `Deepak Cherian <https://github.com/dcherian>`_.
+- Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4.
   By `Anderson Banihirwe <https://github.com/andersy005>`_.
-
+- Fix :py:meth:`xarray.core.groupby.DataArrayGroupBy.reduce` and
+  :py:meth:`xarray.core.groupby.DatasetGroupBy.reduce` when reducing over multiple dimensions.
+  (:issue:`3402`). By `Deepak Cherian <https://github.com/dcherian/>`_
 
 Documentation
 ~~~~~~~~~~~~~

setup.cfg

@@ -117,4 +117,7 @@ tag_prefix = v
 parentdir_prefix = xarray-
 
 [aliases]
-test = pytest
+test = pytest
+
+[pytest-watch]
+nobeep = True

xarray/core/dataarray.py

@@ -1863,12 +1863,7 @@ def transpose(self, *dims: Hashable, transpose_coords: bool = None) -> "DataArra
         Dataset.transpose
         """
         if dims:
-            if set(dims) ^ set(self.dims):
-                raise ValueError(
-                    "arguments to transpose (%s) must be "
-                    "permuted array dimensions (%s)" % (dims, tuple(self.dims))
-                )
-
+            dims = tuple(utils.infix_dims(dims, self.dims))
         variable = self.variable.transpose(*dims)
         if transpose_coords:
             coords: Dict[Hashable, Variable] = {}

xarray/core/dataset.py

@@ -3542,7 +3542,6 @@ def drop(  # noqa: F811
         ----------
         labels : hashable or iterable of hashables
             Name(s) of variables or index labels to drop.
-            If dim is not None, labels can be any array-like.
         dim : None or hashable, optional
             Dimension along which to drop index labels. By default (if
             ``dim is None``), drops variables rather than index labels.
@@ -3712,14 +3711,14 @@ def transpose(self, *dims: Hashable) -> "Dataset":
         DataArray.transpose
         """
         if dims:
-            if set(dims) ^ set(self.dims):
+            if set(dims) ^ set(self.dims) and ... not in dims:
                 raise ValueError(
                     "arguments to transpose (%s) must be "
                     "permuted dataset dimensions (%s)" % (dims, tuple(self.dims))
                 )
         ds = self.copy()
         for name, var in self._variables.items():
-            var_dims = tuple(dim for dim in dims if dim in var.dims)
+            var_dims = tuple(dim for dim in dims if dim in (var.dims + (...,)))
             ds._variables[name] = var.transpose(*var_dims)
         return ds
 

xarray/core/groupby.py

@@ -15,13 +15,26 @@
 from .utils import (
     either_dict_or_kwargs,
     hashable,
+    is_scalar,
     maybe_wrap_array,
     peek_at,
     safe_cast_to_index,
 )
 from .variable import IndexVariable, Variable, as_variable
 
 
+def check_reduce_dims(reduce_dims, dimensions):
+
+    if reduce_dims is not ...:
+        if is_scalar(reduce_dims):
+            reduce_dims = [reduce_dims]
+        if any([dim not in dimensions for dim in reduce_dims]):
+            raise ValueError(
+                "cannot reduce over dimensions %r. expected either '...' to reduce over all dimensions or one or more of %r."
+                % (reduce_dims, dimensions)
+            )
+
+
 def unique_value_groups(ar, sort=True):
     """Group an array by its unique values.
 
@@ -348,6 +361,13 @@ def __init__(
                 group_indices = [slice(i, i + 1) for i in group_indices]
             unique_coord = group
         else:
+            if group.isnull().any():
+                # drop any NaN valued groups.
+                # also drop obj values where group was NaN
+                # Use where instead of reindex to account for duplicate coordinate labels.
+                obj = obj.where(group.notnull(), drop=True)
+                group = group.dropna(group_dim)
+
             # look through group to find the unique values
             unique_values, group_indices = unique_value_groups(
                 safe_cast_to_index(group), sort=(bins is None)
@@ -794,15 +814,11 @@ def reduce(
         if keep_attrs is None:
             keep_attrs = _get_keep_attrs(default=False)
 
-        if dim is not ... and dim not in self.dims:
-            raise ValueError(
-                "cannot reduce over dimension %r. expected either '...' to reduce over all dimensions or one or more of %r."
-                % (dim, self.dims)
-            )
-
         def reduce_array(ar):
             return ar.reduce(func, dim, axis, keep_attrs=keep_attrs, **kwargs)
 
+        check_reduce_dims(dim, self.dims)
+
         return self.apply(reduce_array, shortcut=shortcut)
 
 
@@ -895,11 +911,7 @@ def reduce(self, func, dim=None, keep_attrs=None, **kwargs):
         def reduce_dataset(ds):
             return ds.reduce(func, dim, keep_attrs, **kwargs)
 
-        if dim is not ... and dim not in self.dims:
-            raise ValueError(
-                "cannot reduce over dimension %r. expected either '...' to reduce over all dimensions or one or more of %r."
-                % (dim, self.dims)
-            )
+        check_reduce_dims(dim, self.dims)
 
         return self.apply(reduce_dataset)
 

xarray/core/utils.py

@@ -10,6 +10,7 @@
     AbstractSet,
     Any,
     Callable,
+    Collection,
     Container,
     Dict,
     Hashable,
@@ -660,6 +661,30 @@ def __len__(self) -> int:
         return len(self._data) - num_hidden
 
 
+def infix_dims(dims_supplied: Collection, dims_all: Collection) -> Iterator:
+    """
+    Resolves a supplied list containing an ellispsis representing other items, to
+    a generator with the 'realized' list of all items
+    """
+    if ... in dims_supplied:
+        if len(set(dims_all)) != len(dims_all):
+            raise ValueError("Cannot use ellipsis with repeated dims")
+        if len([d for d in dims_supplied if d == ...]) > 1:
+            raise ValueError("More than one ellipsis supplied")
+        other_dims = [d for d in dims_all if d not in dims_supplied]
+        for d in dims_supplied:
+            if d == ...:
+                yield from other_dims
+            else:
+                yield d
+    else:
+        if set(dims_supplied) ^ set(dims_all):
+            raise ValueError(
+                f"{dims_supplied} must be a permuted list of {dims_all}, unless `...` is included"
+            )
+        yield from dims_supplied
+
+
 def get_temp_dimname(dims: Container[Hashable], new_dim: Hashable) -> Hashable:
     """ Get an new dimension name based on new_dim, that is not used in dims.
     If the same name exists, we add an underscore(s) in the head.

xarray/core/variable.py

@@ -25,6 +25,7 @@
     OrderedSet,
     decode_numpy_dict_values,
     either_dict_or_kwargs,
+    infix_dims,
     ensure_us_time_resolution,
 )
 
@@ -1228,6 +1229,7 @@ def transpose(self, *dims) -> "Variable":
         """
         if len(dims) == 0:
             dims = self.dims[::-1]
+        dims = tuple(infix_dims(dims, self.dims))
         axes = self.get_axis_num(dims)
         if len(dims) < 2 or dims == self.dims:
             # no need to transpose if only one dimension

xarray/static/css/style.css

@@ -2,6 +2,17 @@
  *
  */
 
+:root {
+  --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));
+  --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));
+  --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));
+  --xr-border-color: var(--jp-border-color2, #e0e0e0);
+  --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);
+  --xr-background-color: var(--jp-layout-color0, white);
+  --xr-background-color-row-even: var(--jp-layout-color1, white);
+  --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);
+}
+
 .xr-wrap {
   min-width: 300px;
   max-width: 700px;
@@ -11,7 +22,7 @@
   padding-top: 6px;
   padding-bottom: 6px;
   margin-bottom: 4px;
-  border-bottom: solid 1px #ddd;
+  border-bottom: solid 1px var(--xr-border-color);
 }
 
 .xr-header > div,
@@ -28,11 +39,7 @@
 }
 
 .xr-obj-type {
-  color: #555;
-}
-
-.xr-array-name {
-  color: #000;
+  color: var(--xr-font-color2);
 }
 
 .xr-sections {
@@ -50,21 +57,21 @@
 }
 
 .xr-section-item input + label {
-  color: #ccc;
+  color: var(--xr-disabled-color);
 }
 
 .xr-section-item input:enabled + label {
   cursor: pointer;
-  color: #555;
+  color: var(--xr-font-color2);
 }
 
 .xr-section-item input:enabled + label:hover {
-  color: #000;
+  color: var(--xr-font-color0);
 }
 
 .xr-section-summary {
   grid-column: 1;
-  color: #555;
+  color: var(--xr-font-color2);
   font-weight: 500;
 }
 
@@ -74,7 +81,7 @@
 }
 
 .xr-section-summary-in:disabled + label {
-  color: #555;
+  color: var(--xr-font-color2);
 }
 
 .xr-section-summary-in + label:before {
@@ -86,7 +93,7 @@
 }
 
 .xr-section-summary-in:disabled + label:before {
-  color: #ccc;
+  color: var(--xr-disabled-color);
 }
 
 .xr-section-summary-in:checked + label:before {
@@ -129,7 +136,7 @@
 }
 
 .xr-preview {
-  color: #888;
+  color: var(--xr-font-color3);
 }
 
 .xr-array-preview,
@@ -186,7 +193,7 @@
 .xr-var-item > div,
 .xr-var-item label,
 .xr-var-item > .xr-var-name span {
-  background-color: #fcfcfc;
+  background-color: var(--xr-background-color-row-even);
   margin-bottom: 0;
 }
 
@@ -197,7 +204,7 @@
 .xr-var-list > li:nth-child(odd) > div,
 .xr-var-list > li:nth-child(odd) > label,
 .xr-var-list > li:nth-child(odd) > .xr-var-name span {
-  background-color: #efefef;
+  background-color: var(--xr-background-color-row-odd);
 }
 
 .xr-var-name {
@@ -211,7 +218,7 @@
 .xr-var-dtype {
   grid-column: 3;
   text-align: right;
-  color: #555;
+  color: var(--xr-font-color2);
 }
 
 .xr-var-preview {
@@ -241,7 +248,7 @@
 .xr-var-attrs,
 .xr-var-data {
   display: none;
-  background-color: #fff !important;
+  background-color: var(--xr-background-color) !important;
   padding-bottom: 5px !important;
 }
 
@@ -288,7 +295,7 @@ dl.xr-attrs {
 
 .xr-attrs dt:hover span {
   display: inline-block;
-  background: #fff;
+  background: var(--xr-background-color);
   padding-right: 10px;
 }
 

xarray/tests/__init__.py

@@ -158,18 +158,21 @@ def source_ndarray(array):
 
 
 def assert_equal(a, b):
+    __tracebackhide__ = True
     xarray.testing.assert_equal(a, b)
     xarray.testing._assert_internal_invariants(a)
     xarray.testing._assert_internal_invariants(b)
 
 
 def assert_identical(a, b):
+    __tracebackhide__ = True
     xarray.testing.assert_identical(a, b)
     xarray.testing._assert_internal_invariants(a)
     xarray.testing._assert_internal_invariants(b)
 
 
 def assert_allclose(a, b, **kwargs):
+    __tracebackhide__ = True
     xarray.testing.assert_allclose(a, b, **kwargs)
     xarray.testing._assert_internal_invariants(a)
     xarray.testing._assert_internal_invariants(b)