From ba48fbcd6ee14e0bbd8887a970a1125fde6769f0 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Oct 2019 12:48:45 -0400 Subject: [PATCH 1/5] Html repr (#3425) * add CSS style and internal functions for html repr * move CSS code to its own file in a new static directory * add repr of array objects + some refactoring and fixes * add _repr_html_ methods to dataset, dataarray and variable * fix encoding issue in read CSS * fix some CSS for compatibility with notebook (tested 5.2) * use CSS grid + add icons to show/hide attrs and data repr * Changing title of icons to make tooltips better * Adding option to set repr back to classic * Adding support for multiindexes * Getting rid of some spans and fixing alignment * Forgot to check in css [skip ci] * Overflow on hover * Cleaning up css * Fixing indentation * Replacing + icon with db icon * Unifying input css * Renaming stylesheet [skip ci] * Improving styling of attributes * Using the repr functions * Using dask array _repr_html_ * Fixing alignment of Dimensions * Make sure to include subdirs in package * Adding static to manifest * Trying to include css files * Fixing css discrepancies in colab * Adding in lots of escapes and also f-strings * Adding some tests for formatting_html * linting * classic -> text * linting more * Adding tests for new option * Trying to get better coverage * reformatting * Fixing up test * Last tests hopefully * Fixing dask test to work with lower version * More black * Added what's new section * classic -> text Co-Authored-By: Deepak Cherian * Fixing up dt/dl for jlab * Directly change dl objects for attrs section --- MANIFEST.in | 1 + doc/whats-new.rst | 6 + setup.py | 4 +- xarray/core/common.py | 10 +- xarray/core/dataset.py | 7 + xarray/core/formatting_html.py | 274 ++++++++++++++++++++ xarray/core/options.py | 7 + xarray/static/css/style.css | 310 +++++++++++++++++++++++ xarray/static/html/icons-svg-inline.html | 17 ++ xarray/tests/test_formatting_html.py | 132 ++++++++++ xarray/tests/test_options.py | 37 +++ 11 files changed, 802 insertions(+), 3 deletions(-) mode change 100644 => 100755 setup.py create mode 100644 xarray/core/formatting_html.py create mode 100644 xarray/static/css/style.css create mode 100644 xarray/static/html/icons-svg-inline.html create mode 100644 xarray/tests/test_formatting_html.py diff --git a/MANIFEST.in b/MANIFEST.in index a006660e5fb..4d5c34f622c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,3 +6,4 @@ prune doc/generated global-exclude .DS_Store include versioneer.py include xarray/_version.py +recursive-include xarray/static * diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 9d3e64badb8..12bed8f332e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,6 +36,12 @@ New Features ``pip install git+https://github.com/andrewgsavage/pint.git@refs/pull/6/head)``. Even with it, interaction with non-numpy array libraries, e.g. dask or sparse, is broken. +- Added new :py:meth:`Dataset._repr_html_` and :py:meth:`DataArray._repr_html_` to improve + representation of objects in jupyter. By default this feature is turned off + for now. Enable it with :py:meth:`xarray.set_options(display_style="html")`. + (:pull:`3425`) by `Benoit Bovy `_ and + `Julia Signell `_. + Bug fixes ~~~~~~~~~ - Fix regression introduced in v0.14.0 that would cause a crash if dask is installed diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 08d4f54764f..cba0c74aa3a --- a/setup.py +++ b/setup.py @@ -104,5 +104,7 @@ tests_require=TESTS_REQUIRE, url=URL, packages=find_packages(), - package_data={"xarray": ["py.typed", "tests/data/*"]}, + package_data={ + "xarray": ["py.typed", "tests/data/*", "static/css/*", "static/html/*"] + }, ) diff --git a/xarray/core/common.py b/xarray/core/common.py index 45d860a1797..1a8cf34ed39 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -1,5 +1,6 @@ import warnings from contextlib import suppress +from html import escape from textwrap import dedent from typing import ( Any, @@ -18,10 +19,10 @@ import numpy as np import pandas as pd -from . import dtypes, duck_array_ops, formatting, ops +from . import dtypes, duck_array_ops, formatting, formatting_html, ops from .arithmetic import SupportsArithmetic from .npcompat import DTypeLike -from .options import _get_keep_attrs +from .options import OPTIONS, _get_keep_attrs from .pycompat import dask_array_type from .rolling_exp import RollingExp from .utils import Frozen, ReprObject, either_dict_or_kwargs @@ -134,6 +135,11 @@ def __array__(self: Any, dtype: DTypeLike = None) -> np.ndarray: def __repr__(self) -> str: return formatting.array_repr(self) + def _repr_html_(self): + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return formatting_html.array_repr(self) + def _iter(self: Any) -> Iterator[Any]: for n in range(len(self)): yield self[n] diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 12d5cbdc9f3..eba580f84bd 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3,6 +3,7 @@ import sys import warnings from collections import defaultdict +from html import escape from numbers import Number from pathlib import Path from typing import ( @@ -39,6 +40,7 @@ dtypes, duck_array_ops, formatting, + formatting_html, groupby, ops, resample, @@ -1619,6 +1621,11 @@ def to_zarr( def __repr__(self) -> str: return formatting.dataset_repr(self) + def _repr_html_(self): + if OPTIONS["display_style"] == "text": + return f"
{escape(repr(self))}
" + return formatting_html.dataset_repr(self) + def info(self, buf=None) -> None: """ Concise summary of a Dataset variables and attributes. diff --git a/xarray/core/formatting_html.py b/xarray/core/formatting_html.py new file mode 100644 index 00000000000..b03ecc12962 --- /dev/null +++ b/xarray/core/formatting_html.py @@ -0,0 +1,274 @@ +import uuid +import pkg_resources +from collections import OrderedDict +from functools import partial +from html import escape + +from .formatting import inline_variable_array_repr, short_data_repr + + +CSS_FILE_PATH = "/".join(("static", "css", "style.css")) +CSS_STYLE = pkg_resources.resource_string("xarray", CSS_FILE_PATH).decode("utf8") + + +ICONS_SVG_PATH = "/".join(("static", "html", "icons-svg-inline.html")) +ICONS_SVG = pkg_resources.resource_string("xarray", ICONS_SVG_PATH).decode("utf8") + + +def short_data_repr_html(array): + """Format "data" for DataArray and Variable.""" + internal_data = getattr(array, "variable", array)._data + if hasattr(internal_data, "_repr_html_"): + return internal_data._repr_html_() + return escape(short_data_repr(array)) + + +def format_dims(dims, coord_names): + if not dims: + return "" + + dim_css_map = { + k: " class='xr-has-index'" if k in coord_names else "" for k, v in dims.items() + } + + dims_li = "".join( + f"
  • " f"{escape(dim)}: {size}
  • " + for dim, size in dims.items() + ) + + return f"
      {dims_li}
    " + + +def summarize_attrs(attrs): + attrs_dl = "".join( + f"
    {escape(k)} :
    " f"
    {escape(str(v))}
    " + for k, v in attrs.items() + ) + + return f"
    {attrs_dl}
    " + + +def _icon(icon_name): + # icon_name should be defined in xarray/static/html/icon-svg-inline.html + return ( + "" + "" + "" + "".format(icon_name) + ) + + +def _summarize_coord_multiindex(name, coord): + preview = f"({', '.join(escape(l) for l in coord.level_names)})" + return summarize_variable( + name, coord, is_index=True, dtype="MultiIndex", preview=preview + ) + + +def summarize_coord(name, var): + is_index = name in var.dims + if is_index: + coord = var.variable.to_index_variable() + if coord.level_names is not None: + coords = {} + coords[name] = _summarize_coord_multiindex(name, coord) + for lname in coord.level_names: + var = coord.get_level_variable(lname) + coords[lname] = summarize_variable(lname, var) + return coords + + return {name: summarize_variable(name, var, is_index)} + + +def summarize_coords(variables): + coords = {} + for k, v in variables.items(): + coords.update(**summarize_coord(k, v)) + + vars_li = "".join(f"
  • {v}
  • " for v in coords.values()) + + return f"
      {vars_li}
    " + + +def summarize_variable(name, var, is_index=False, dtype=None, preview=None): + variable = var.variable if hasattr(var, "variable") else var + + cssclass_idx = " class='xr-has-index'" if is_index else "" + dims_str = f"({', '.join(escape(dim) for dim in var.dims)})" + name = escape(name) + dtype = dtype or var.dtype + + # "unique" ids required to expand/collapse subsections + attrs_id = "attrs-" + str(uuid.uuid4()) + data_id = "data-" + str(uuid.uuid4()) + disabled = "" if len(var.attrs) else "disabled" + + preview = preview or escape(inline_variable_array_repr(variable, 35)) + attrs_ul = summarize_attrs(var.attrs) + data_repr = short_data_repr_html(variable) + + attrs_icon = _icon("icon-file-text2") + data_icon = _icon("icon-database") + + return ( + f"
    {name}
    " + f"
    {dims_str}
    " + f"
    {dtype}
    " + f"
    {preview}
    " + f"" + f"" + f"" + f"" + f"
    {attrs_ul}
    " + f"
    {data_repr}
    " + ) + + +def summarize_vars(variables): + vars_li = "".join( + f"
  • {summarize_variable(k, v)}
  • " + for k, v in variables.items() + ) + + return f"
      {vars_li}
    " + + +def collapsible_section( + name, inline_details="", details="", n_items=None, enabled=True, collapsed=False +): + # "unique" id to expand/collapse the section + data_id = "section-" + str(uuid.uuid4()) + + has_items = n_items is not None and n_items + n_items_span = "" if n_items is None else f" ({n_items})" + enabled = "" if enabled and has_items else "disabled" + collapsed = "" if collapsed or not has_items else "checked" + tip = " title='Expand/collapse section'" if enabled else "" + + return ( + f"" + f"" + f"
    {inline_details}
    " + f"
    {details}
    " + ) + + +def _mapping_section(mapping, name, details_func, max_items_collapse, enabled=True): + n_items = len(mapping) + collapsed = n_items >= max_items_collapse + + return collapsible_section( + name, + details=details_func(mapping), + n_items=n_items, + enabled=enabled, + collapsed=collapsed, + ) + + +def dim_section(obj): + dim_list = format_dims(obj.dims, list(obj.coords)) + + return collapsible_section( + "Dimensions", inline_details=dim_list, enabled=False, collapsed=True + ) + + +def array_section(obj): + # "unique" id to expand/collapse the section + data_id = "section-" + str(uuid.uuid4()) + collapsed = "" + preview = escape(inline_variable_array_repr(obj.variable, max_width=70)) + data_repr = short_data_repr_html(obj) + data_icon = _icon("icon-database") + + return ( + "
    " + f"" + f"" + f"
    {preview}
    " + f"
    {data_repr}
    " + "
    " + ) + + +coord_section = partial( + _mapping_section, + name="Coordinates", + details_func=summarize_coords, + max_items_collapse=25, +) + + +datavar_section = partial( + _mapping_section, + name="Data variables", + details_func=summarize_vars, + max_items_collapse=15, +) + + +attr_section = partial( + _mapping_section, + name="Attributes", + details_func=summarize_attrs, + max_items_collapse=10, +) + + +def _obj_repr(header_components, sections): + header = f"
    {''.join(h for h in header_components)}
    " + sections = "".join(f"
  • {s}
  • " for s in sections) + + return ( + "
    " + f"{ICONS_SVG}" + "
    " + f"{header}" + f"
      {sections}
    " + "
    " + "
    " + ) + + +def array_repr(arr): + dims = OrderedDict((k, v) for k, v in zip(arr.dims, arr.shape)) + + obj_type = "xarray.{}".format(type(arr).__name__) + arr_name = "'{}'".format(arr.name) if getattr(arr, "name", None) else "" + coord_names = list(arr.coords) if hasattr(arr, "coords") else [] + + header_components = [ + "
    {}
    ".format(obj_type), + "
    {}
    ".format(arr_name), + format_dims(dims, coord_names), + ] + + sections = [array_section(arr)] + + if hasattr(arr, "coords"): + sections.append(coord_section(arr.coords)) + + sections.append(attr_section(arr.attrs)) + + return _obj_repr(header_components, sections) + + +def dataset_repr(ds): + obj_type = "xarray.{}".format(type(ds).__name__) + + header_components = [f"
    {escape(obj_type)}
    "] + + sections = [ + dim_section(ds), + coord_section(ds.coords), + datavar_section(ds.data_vars), + attr_section(ds.attrs), + ] + + return _obj_repr(header_components, sections) diff --git a/xarray/core/options.py b/xarray/core/options.py index 2f464a33fb1..72f9ad8e1fa 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -8,6 +8,7 @@ CMAP_SEQUENTIAL = "cmap_sequential" CMAP_DIVERGENT = "cmap_divergent" KEEP_ATTRS = "keep_attrs" +DISPLAY_STYLE = "display_style" OPTIONS = { @@ -19,9 +20,11 @@ CMAP_SEQUENTIAL: "viridis", CMAP_DIVERGENT: "RdBu_r", KEEP_ATTRS: "default", + DISPLAY_STYLE: "text", } _JOIN_OPTIONS = frozenset(["inner", "outer", "left", "right", "exact"]) +_DISPLAY_OPTIONS = frozenset(["text", "html"]) def _positive_integer(value): @@ -35,6 +38,7 @@ def _positive_integer(value): FILE_CACHE_MAXSIZE: _positive_integer, WARN_FOR_UNCLOSED_FILES: lambda value: isinstance(value, bool), KEEP_ATTRS: lambda choice: choice in [True, False, "default"], + DISPLAY_STYLE: _DISPLAY_OPTIONS.__contains__, } @@ -98,6 +102,9 @@ class set_options: attrs, ``False`` to always discard them, or ``'default'`` to use original logic that attrs should only be kept in unambiguous circumstances. Default: ``'default'``. + - ``display_style``: display style to use in jupyter for xarray objects. + Default: ``'text'``. Other options are ``'html'``. + You can use ``set_options`` either as a context manager: diff --git a/xarray/static/css/style.css b/xarray/static/css/style.css new file mode 100644 index 00000000000..536b8ab6103 --- /dev/null +++ b/xarray/static/css/style.css @@ -0,0 +1,310 @@ +/* CSS stylesheet for displaying xarray objects in jupyterlab. + * + */ + +.xr-wrap { + min-width: 300px; + max-width: 700px; +} + +.xr-header { + padding-top: 6px; + padding-bottom: 6px; + margin-bottom: 4px; + border-bottom: solid 1px #ddd; +} + +.xr-header > div, +.xr-header > ul { + display: inline; + margin-top: 0; + margin-bottom: 0; +} + +.xr-obj-type, +.xr-array-name { + margin-left: 2px; + margin-right: 10px; +} + +.xr-obj-type { + color: #555; +} + +.xr-array-name { + color: #000; +} + +.xr-sections { + padding-left: 0 !important; + display: grid; + grid-template-columns: 150px auto auto 1fr 20px 20px; +} + +.xr-section-item { + display: contents; +} + +.xr-section-item input { + display: none; +} + +.xr-section-item input + label { + color: #ccc; +} + +.xr-section-item input:enabled + label { + cursor: pointer; + color: #555; +} + +.xr-section-item input:enabled + label:hover { + color: #000; +} + +.xr-section-summary { + grid-column: 1; + color: #555; + font-weight: 500; +} + +.xr-section-summary > span { + display: inline-block; + padding-left: 0.5em; +} + +.xr-section-summary-in:disabled + label { + color: #555; +} + +.xr-section-summary-in + label:before { + display: inline-block; + content: '►'; + font-size: 11px; + width: 15px; + text-align: center; +} + +.xr-section-summary-in:disabled + label:before { + color: #ccc; +} + +.xr-section-summary-in:checked + label:before { + content: '▼'; +} + +.xr-section-summary-in:checked + label > span { + display: none; +} + +.xr-section-summary, +.xr-section-inline-details { + padding-top: 4px; + padding-bottom: 4px; +} + +.xr-section-inline-details { + grid-column: 2 / -1; +} + +.xr-section-details { + display: none; + grid-column: 1 / -1; + margin-bottom: 5px; +} + +.xr-section-summary-in:checked ~ .xr-section-details { + display: contents; +} + +.xr-array-wrap { + grid-column: 1 / -1; + display: grid; + grid-template-columns: 20px auto; +} + +.xr-array-wrap > label { + grid-column: 1; + vertical-align: top; +} + +.xr-preview { + color: #888; +} + +.xr-array-preview, +.xr-array-data { + padding: 0 5px !important; + grid-column: 2; +} + +.xr-array-data, +.xr-array-in:checked ~ .xr-array-preview { + display: none; +} + +.xr-array-in:checked ~ .xr-array-data, +.xr-array-preview { + display: inline-block; +} + +.xr-dim-list { + display: inline-block !important; + list-style: none; + padding: 0 !important; + margin: 0; +} + +.xr-dim-list li { + display: inline-block; + padding: 0; + margin: 0; +} + +.xr-dim-list:before { + content: '('; +} + +.xr-dim-list:after { + content: ')'; +} + +.xr-dim-list li:not(:last-child):after { + content: ','; + padding-right: 5px; +} + +.xr-has-index { + font-weight: bold; +} + +.xr-var-list, +.xr-var-item { + display: contents; +} + +.xr-var-item > div, +.xr-var-item label, +.xr-var-item > .xr-var-name span { + background-color: #fcfcfc; + margin-bottom: 0; +} + +.xr-var-item > .xr-var-name:hover span { + padding-right: 5px; +} + +.xr-var-list > li:nth-child(odd) > div, +.xr-var-list > li:nth-child(odd) > label, +.xr-var-list > li:nth-child(odd) > .xr-var-name span { + background-color: #efefef; +} + +.xr-var-name { + grid-column: 1; +} + +.xr-var-dims { + grid-column: 2; +} + +.xr-var-dtype { + grid-column: 3; + text-align: right; + color: #555; +} + +.xr-var-preview { + grid-column: 4; +} + +.xr-var-name, +.xr-var-dims, +.xr-var-dtype, +.xr-preview, +.xr-attrs dt { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + padding-right: 10px; +} + +.xr-var-name:hover, +.xr-var-dims:hover, +.xr-var-dtype:hover, +.xr-attrs dt:hover { + overflow: visible; + width: auto; + z-index: 1; +} + +.xr-var-attrs, +.xr-var-data { + display: none; + background-color: #fff !important; + padding-bottom: 5px !important; +} + +.xr-var-attrs-in:checked ~ .xr-var-attrs, +.xr-var-data-in:checked ~ .xr-var-data { + display: block; +} + +.xr-var-data > table { + float: right; +} + +.xr-var-name span, +.xr-var-data, +.xr-attrs { + padding-left: 25px !important; +} + +.xr-attrs, +.xr-var-attrs, +.xr-var-data { + grid-column: 1 / -1; +} + +dl.xr-attrs { + padding: 0; + margin: 0; + display: grid; + grid-template-columns: 125px auto; +} + +.xr-attrs dt, dd { + padding: 0; + margin: 0; + float: left; + padding-right: 10px; + width: auto; +} + +.xr-attrs dt { + font-weight: normal; + grid-column: 1; +} + +.xr-attrs dt:hover span { + display: inline-block; + background: #fff; + padding-right: 10px; +} + +.xr-attrs dd { + grid-column: 2; + white-space: pre-wrap; + word-break: break-all; +} + +.xr-icon-database, +.xr-icon-file-text2 { + display: inline-block; + vertical-align: middle; + width: 1em; + height: 1.5em !important; + stroke-width: 0; + stroke: currentColor; + fill: currentColor; +} diff --git a/xarray/static/html/icons-svg-inline.html b/xarray/static/html/icons-svg-inline.html new file mode 100644 index 00000000000..c44f89c4304 --- /dev/null +++ b/xarray/static/html/icons-svg-inline.html @@ -0,0 +1,17 @@ + + + +Show/Hide data repr + + + + + +Show/Hide attributes + + + + + + + diff --git a/xarray/tests/test_formatting_html.py b/xarray/tests/test_formatting_html.py new file mode 100644 index 00000000000..e7f54b22d06 --- /dev/null +++ b/xarray/tests/test_formatting_html.py @@ -0,0 +1,132 @@ +from distutils.version import LooseVersion + +import numpy as np +import pandas as pd +import pytest + +import xarray as xr +from xarray.core import formatting_html as fh + + +@pytest.fixture +def dataarray(): + return xr.DataArray(np.random.RandomState(0).randn(4, 6)) + + +@pytest.fixture +def dask_dataarray(dataarray): + pytest.importorskip("dask") + return dataarray.chunk() + + +@pytest.fixture +def multiindex(): + mindex = pd.MultiIndex.from_product( + [["a", "b"], [1, 2]], names=("level_1", "level_2") + ) + return xr.Dataset({}, {"x": mindex}) + + +@pytest.fixture +def dataset(): + times = pd.date_range("2000-01-01", "2001-12-31", name="time") + annual_cycle = np.sin(2 * np.pi * (times.dayofyear.values / 365.25 - 0.28)) + + base = 10 + 15 * annual_cycle.reshape(-1, 1) + tmin_values = base + 3 * np.random.randn(annual_cycle.size, 3) + tmax_values = base + 10 + 3 * np.random.randn(annual_cycle.size, 3) + + return xr.Dataset( + { + "tmin": (("time", "location"), tmin_values), + "tmax": (("time", "location"), tmax_values), + }, + {"time": times, "location": ["", "IN", "IL"]}, + attrs={"description": "Test data."}, + ) + + +def test_short_data_repr_html(dataarray): + data_repr = fh.short_data_repr_html(dataarray) + assert data_repr.startswith("array") + + +def test_short_data_repr_html_dask(dask_dataarray): + import dask + + if LooseVersion(dask.__version__) < "2.0.0": + assert not hasattr(dask_dataarray.data, "_repr_html_") + data_repr = fh.short_data_repr_html(dask_dataarray) + assert ( + data_repr + == "dask.array<xarray-<this-array>, shape=(4, 6), dtype=float64, chunksize=(4, 6)>" + ) + else: + assert hasattr(dask_dataarray.data, "_repr_html_") + data_repr = fh.short_data_repr_html(dask_dataarray) + assert data_repr == dask_dataarray.data._repr_html_() + + +def test_format_dims_no_dims(): + dims, coord_names = {}, [] + formatted = fh.format_dims(dims, coord_names) + assert formatted == "" + + +def test_format_dims_unsafe_dim_name(): + dims, coord_names = {"": 3, "y": 2}, [] + formatted = fh.format_dims(dims, coord_names) + assert "<x>" in formatted + + +def test_format_dims_non_index(): + dims, coord_names = {"x": 3, "y": 2}, ["time"] + formatted = fh.format_dims(dims, coord_names) + assert "class='xr-has-index'" not in formatted + + +def test_format_dims_index(): + dims, coord_names = {"x": 3, "y": 2}, ["x"] + formatted = fh.format_dims(dims, coord_names) + assert "class='xr-has-index'" in formatted + + +def test_summarize_attrs_with_unsafe_attr_name_and_value(): + attrs = {"": 3, "y": ""} + formatted = fh.summarize_attrs(attrs) + assert "
    <x> :
    " in formatted + assert "
    y :
    " in formatted + assert "
    3
    " in formatted + assert "
    <pd.DataFrame>
    " in formatted + + +def test_repr_of_dataarray(dataarray): + formatted = fh.array_repr(dataarray) + assert "dim_0" in formatted + # has an expandable data section + assert formatted.count("class='xr-array-in' type='checkbox' >") == 1 + # coords and attrs don't have an items so they'll be be disabled and collapsed + assert ( + formatted.count("class='xr-section-summary-in' type='checkbox' disabled >") == 2 + ) + + +def test_summary_of_multiindex_coord(multiindex): + idx = multiindex.x.variable.to_index_variable() + formatted = fh._summarize_coord_multiindex("foo", idx) + assert "(level_1, level_2)" in formatted + assert "MultiIndex" in formatted + assert "foo" in formatted + + +def test_repr_of_multiindex(multiindex): + formatted = fh.dataset_repr(multiindex) + assert "(x)" in formatted + + +def test_repr_of_dataset(dataset): + formatted = fh.dataset_repr(dataset) + # coords, attrs, and data_vars are expanded + assert ( + formatted.count("class='xr-section-summary-in' type='checkbox' checked>") == 3 + ) diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index 2aa77ecd6b3..f155acbf494 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -67,6 +67,16 @@ def test_nested_options(): assert OPTIONS["display_width"] == original +def test_display_style(): + original = "text" + assert OPTIONS["display_style"] == original + with pytest.raises(ValueError): + xarray.set_options(display_style="invalid_str") + with xarray.set_options(display_style="html"): + assert OPTIONS["display_style"] == "html" + assert OPTIONS["display_style"] == original + + def create_test_dataset_attrs(seed=0): ds = create_test_data(seed) ds.attrs = {"attr1": 5, "attr2": "history", "attr3": {"nested": "more_info"}} @@ -164,3 +174,30 @@ def test_merge_attr_retention(self): # option doesn't affect this result = merge([da1, da2]) assert result.attrs == original_attrs + + def test_display_style_text(self): + ds = create_test_dataset_attrs() + text = ds._repr_html_() + assert text.startswith("
    ")
    +        assert "'nested'" in text
    +        assert "<xarray.Dataset>" in text
    +
    +    def test_display_style_html(self):
    +        ds = create_test_dataset_attrs()
    +        with xarray.set_options(display_style="html"):
    +            html = ds._repr_html_()
    +            assert html.startswith("
    ") + assert "'nested'" in html + + def test_display_dataarray_style_text(self): + da = create_test_dataarray_attrs() + text = da._repr_html_() + assert text.startswith("
    ")
    +        assert "<xarray.DataArray 'var1'" in text
    +
    +    def test_display_dataarray_style_html(self):
    +        da = create_test_dataarray_attrs()
    +        with xarray.set_options(display_style="html"):
    +            html = da._repr_html_()
    +            assert html.startswith("
    ") + assert "#x27;nested'" in html From bb0a5a2b1c71f7c2622543406ccc82ddbb290ece Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Oct 2019 17:50:19 -0400 Subject: [PATCH 2/5] Escaping dtypes (#3444) * Escaping dtypes * Reformatting --- xarray/core/formatting_html.py | 2 +- xarray/tests/test_formatting_html.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/core/formatting_html.py b/xarray/core/formatting_html.py index b03ecc12962..dbebbcf4fbe 100644 --- a/xarray/core/formatting_html.py +++ b/xarray/core/formatting_html.py @@ -96,7 +96,7 @@ def summarize_variable(name, var, is_index=False, dtype=None, preview=None): cssclass_idx = " class='xr-has-index'" if is_index else "" dims_str = f"({', '.join(escape(dim) for dim in var.dims)})" name = escape(name) - dtype = dtype or var.dtype + dtype = dtype or escape(str(var.dtype)) # "unique" ids required to expand/collapse subsections attrs_id = "attrs-" + str(uuid.uuid4()) diff --git a/xarray/tests/test_formatting_html.py b/xarray/tests/test_formatting_html.py index e7f54b22d06..fea24ff93f8 100644 --- a/xarray/tests/test_formatting_html.py +++ b/xarray/tests/test_formatting_html.py @@ -130,3 +130,5 @@ def test_repr_of_dataset(dataset): assert ( formatted.count("class='xr-section-summary-in' type='checkbox' checked>") == 3 ) + assert "<U4" in formatted + assert "<IA>" in formatted From 79b3cdd3822c79ad2ee267f4d5082cd91c7f714c Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Fri, 25 Oct 2019 11:15:46 -0400 Subject: [PATCH 3/5] change ALL_DIMS to equal ellipsis (#3418) * change ALL_DIMS to equal ... * changed references & added whatsnew * Update xarray/core/groupby.py Co-Authored-By: Deepak Cherian * Update xarray/core/groupby.py Co-Authored-By: Deepak Cherian * note in readme --- doc/examples/multidimensional-coords.rst | 2 +- doc/groupby.rst | 16 +++++++++++----- doc/whats-new.rst | 5 +++++ xarray/core/common.py | 4 ++-- xarray/core/dataset.py | 5 ++--- xarray/core/groupby.py | 16 ++++++++-------- xarray/core/variable.py | 2 +- xarray/tests/test_dask.py | 4 ++-- xarray/tests/test_dataarray.py | 14 +++++++------- xarray/tests/test_dataset.py | 13 ++++++------- xarray/tests/test_groupby.py | 6 +++--- xarray/tests/test_plot.py | 6 +++--- xarray/tests/test_sparse.py | 8 ++++---- 13 files changed, 55 insertions(+), 46 deletions(-) diff --git a/doc/examples/multidimensional-coords.rst b/doc/examples/multidimensional-coords.rst index a5084043977..55569b7662a 100644 --- a/doc/examples/multidimensional-coords.rst +++ b/doc/examples/multidimensional-coords.rst @@ -107,7 +107,7 @@ function to specify the output coordinates of the group. lat_center = np.arange(1, 90, 2) # group according to those bins and take the mean Tair_lat_mean = (ds.Tair.groupby_bins('xc', lat_bins, labels=lat_center) - .mean(xr.ALL_DIMS)) + .mean(...)) # plot the result @savefig xarray_multidimensional_coords_14_1.png width=5in Tair_lat_mean.plot(); diff --git a/doc/groupby.rst b/doc/groupby.rst index e1d88e289d2..52a27f4f160 100644 --- a/doc/groupby.rst +++ b/doc/groupby.rst @@ -116,7 +116,13 @@ dimensions *other than* the provided one: .. ipython:: python - ds.groupby('x').std(xr.ALL_DIMS) + ds.groupby('x').std(...) + +.. note:: + + We use an ellipsis (`...`) here to indicate we want to reduce over all + other dimensions + First and last ~~~~~~~~~~~~~~ @@ -127,7 +133,7 @@ values for group along the grouped dimension: .. ipython:: python - ds.groupby('letters').first(xr.ALL_DIMS) + ds.groupby('letters').first(...) By default, they skip missing values (control this with ``skipna``). @@ -142,7 +148,7 @@ coordinates. For example: .. ipython:: python - alt = arr.groupby('letters').mean(xr.ALL_DIMS) + alt = arr.groupby('letters').mean(...) alt ds.groupby('letters') - alt @@ -195,7 +201,7 @@ __ http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#_two_dimen 'lat': (['ny','nx'], [[10,10],[20,20]] ),}, dims=['ny','nx']) da - da.groupby('lon').sum(xr.ALL_DIMS) + da.groupby('lon').sum(...) da.groupby('lon').apply(lambda x: x - x.mean(), shortcut=False) Because multidimensional groups have the ability to generate a very large @@ -213,4 +219,4 @@ applying your function, and then unstacking the result: .. ipython:: python stacked = da.stack(gridcell=['ny', 'nx']) - stacked.groupby('gridcell').sum(xr.ALL_DIMS).unstack('gridcell') + stacked.groupby('gridcell').sum(...).unstack('gridcell') diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 12bed8f332e..ac60994d35b 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -25,6 +25,11 @@ Breaking changes New Features ~~~~~~~~~~~~ +- Changed `xr.ALL_DIMS` to equal python's `Ellipsis` (`...`), and changed internal usages to use + `...` directly. As before, you can use this to instruct a `groupby` operation + to reduce over all dimensions. While we have no plans to remove `xr.ALL_DIMS`, we suggest + using `...`. + By `Maximilian Roos `_ - Added integration tests against `pint `_. (:pull:`3238`) by `Justus Magin `_. diff --git a/xarray/core/common.py b/xarray/core/common.py index 1a8cf34ed39..d372115ea57 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -25,10 +25,10 @@ from .options import OPTIONS, _get_keep_attrs from .pycompat import dask_array_type from .rolling_exp import RollingExp -from .utils import Frozen, ReprObject, either_dict_or_kwargs +from .utils import Frozen, either_dict_or_kwargs # Used as a sentinel value to indicate a all dimensions -ALL_DIMS = ReprObject("") +ALL_DIMS = ... C = TypeVar("C") diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index eba580f84bd..55ac0bc6135 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -49,7 +49,6 @@ ) from .alignment import _broadcast_helper, _get_broadcast_dims_map_common_coords, align from .common import ( - ALL_DIMS, DataWithCoords, ImplementsDatasetReduce, _contains_datetime_like_objects, @@ -4037,7 +4036,7 @@ def reduce( Dataset with this object's DataArrays replaced with new DataArrays of summarized data and the indicated dimension(s) removed. """ - if dim is None or dim is ALL_DIMS: + if dim is None or dim is ...: dims = set(self.dims) elif isinstance(dim, str) or not isinstance(dim, Iterable): dims = {dim} @@ -5002,7 +5001,7 @@ def quantile( if isinstance(dim, str): dims = {dim} - elif dim is None or dim is ALL_DIMS: + elif dim in [None, ...]: dims = set(self.dims) else: dims = set(dim) diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index 52eb17df18d..68bd28ddb12 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -7,7 +7,7 @@ from . import dtypes, duck_array_ops, nputils, ops from .arithmetic import SupportsArithmetic -from .common import ALL_DIMS, ImplementsArrayReduce, ImplementsDatasetReduce +from .common import ImplementsArrayReduce, ImplementsDatasetReduce from .concat import concat from .formatting import format_array_flat from .options import _get_keep_attrs @@ -712,7 +712,7 @@ def quantile(self, q, dim=None, interpolation="linear", keep_attrs=None): q : float in range of [0,1] (or sequence of floats) Quantile to compute, which must be between 0 and 1 inclusive. - dim : xarray.ALL_DIMS, str or sequence of str, optional + dim : `...`, str or sequence of str, optional Dimension(s) over which to apply quantile. Defaults to the grouped dimension. interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'} @@ -769,7 +769,7 @@ def reduce( Function which can be called in the form `func(x, axis=axis, **kwargs)` to return the result of collapsing an np.ndarray over an integer valued axis. - dim : xarray.ALL_DIMS, str or sequence of str, optional + dim : `...`, str or sequence of str, optional Dimension(s) over which to apply `func`. axis : int or sequence of int, optional Axis(es) over which to apply `func`. Only one of the 'dimension' @@ -794,9 +794,9 @@ def reduce( if keep_attrs is None: keep_attrs = _get_keep_attrs(default=False) - if dim is not ALL_DIMS and dim not in self.dims: + if dim is not ... and dim not in self.dims: raise ValueError( - "cannot reduce over dimension %r. expected either xarray.ALL_DIMS to reduce over all dimensions or one or more of %r." + "cannot reduce over dimension %r. expected either '...' to reduce over all dimensions or one or more of %r." % (dim, self.dims) ) @@ -867,7 +867,7 @@ def reduce(self, func, dim=None, keep_attrs=None, **kwargs): Function which can be called in the form `func(x, axis=axis, **kwargs)` to return the result of collapsing an np.ndarray over an integer valued axis. - dim : xarray.ALL_DIMS, str or sequence of str, optional + dim : `...`, str or sequence of str, optional Dimension(s) over which to apply `func`. axis : int or sequence of int, optional Axis(es) over which to apply `func`. Only one of the 'dimension' @@ -895,9 +895,9 @@ def reduce(self, func, dim=None, keep_attrs=None, **kwargs): def reduce_dataset(ds): return ds.reduce(func, dim, keep_attrs, **kwargs) - if dim is not ALL_DIMS and dim not in self.dims: + if dim is not ... and dim not in self.dims: raise ValueError( - "cannot reduce over dimension %r. expected either xarray.ALL_DIMS to reduce over all dimensions or one or more of %r." + "cannot reduce over dimension %r. expected either '...' to reduce over all dimensions or one or more of %r." % (dim, self.dims) ) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 37672cd82d9..93ad1eafb97 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1450,7 +1450,7 @@ def reduce( Array with summarized data and the indicated dimension(s) removed. """ - if dim is common.ALL_DIMS: + if dim == ...: dim = None if dim is not None and axis is not None: raise ValueError("cannot supply both 'axis' and 'dim' arguments") diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index ae8f43cb66d..50517ae3c9c 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -435,8 +435,8 @@ def test_groupby(self): u = self.eager_array v = self.lazy_array - expected = u.groupby("x").mean(xr.ALL_DIMS) - actual = v.groupby("x").mean(xr.ALL_DIMS) + expected = u.groupby("x").mean(...) + actual = v.groupby("x").mean(...) self.assertLazyAndAllClose(expected, actual) def test_groupby_first(self): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index a3a2f55f6cc..b13527bc098 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -13,7 +13,7 @@ from xarray.coding.times import CFDatetimeCoder from xarray.convert import from_cdms2 from xarray.core import dtypes -from xarray.core.common import ALL_DIMS, full_like +from xarray.core.common import full_like from xarray.tests import ( LooseVersion, ReturnItem, @@ -2443,8 +2443,8 @@ def test_groupby_sum(self): "abc": Variable(["abc"], np.array(["a", "b", "c"])), } )["foo"] - assert_allclose(expected_sum_all, grouped.reduce(np.sum, dim=ALL_DIMS)) - assert_allclose(expected_sum_all, grouped.sum(ALL_DIMS)) + assert_allclose(expected_sum_all, grouped.reduce(np.sum, dim=...)) + assert_allclose(expected_sum_all, grouped.sum(...)) expected = DataArray( [ @@ -2456,7 +2456,7 @@ def test_groupby_sum(self): ) actual = array["y"].groupby("abc").apply(np.sum) assert_allclose(expected, actual) - actual = array["y"].groupby("abc").sum(ALL_DIMS) + actual = array["y"].groupby("abc").sum(...) assert_allclose(expected, actual) expected_sum_axis1 = Dataset( @@ -2590,9 +2590,9 @@ def test_groupby_math(self): assert_identical(expected, actual) grouped = array.groupby("abc") - expected_agg = (grouped.mean(ALL_DIMS) - np.arange(3)).rename(None) + expected_agg = (grouped.mean(...) - np.arange(3)).rename(None) actual = grouped - DataArray(range(3), [("abc", ["a", "b", "c"])]) - actual_agg = actual.groupby("abc").mean(ALL_DIMS) + actual_agg = actual.groupby("abc").mean(...) assert_allclose(expected_agg, actual_agg) with raises_regex(TypeError, "only support binary ops"): @@ -2698,7 +2698,7 @@ def test_groupby_multidim(self): ("lon", DataArray([5, 28, 23], coords=[("lon", [30.0, 40.0, 50.0])])), ("lat", DataArray([16, 40], coords=[("lat", [10.0, 20.0])])), ]: - actual_sum = array.groupby(dim).sum(ALL_DIMS) + actual_sum = array.groupby(dim).sum(...) assert_identical(expected_sum, actual_sum) def test_groupby_multidim_apply(self): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 006d6881b5a..b3ffdf68e3f 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -11,7 +11,6 @@ import xarray as xr from xarray import ( - ALL_DIMS, DataArray, Dataset, IndexVariable, @@ -3327,7 +3326,7 @@ def test_groupby_reduce(self): expected = data.mean("y") expected["yonly"] = expected["yonly"].variable.set_dims({"x": 3}) - actual = data.groupby("x").mean(ALL_DIMS) + actual = data.groupby("x").mean(...) assert_allclose(expected, actual) actual = data.groupby("x").mean("y") @@ -3336,12 +3335,12 @@ def test_groupby_reduce(self): letters = data["letters"] expected = Dataset( { - "xy": data["xy"].groupby(letters).mean(ALL_DIMS), + "xy": data["xy"].groupby(letters).mean(...), "xonly": (data["xonly"].mean().variable.set_dims({"letters": 2})), "yonly": data["yonly"].groupby(letters).mean(), } ) - actual = data.groupby("letters").mean(ALL_DIMS) + actual = data.groupby("letters").mean(...) assert_allclose(expected, actual) def test_groupby_math(self): @@ -3404,14 +3403,14 @@ def test_groupby_math_virtual(self): {"x": ("t", [1, 2, 3])}, {"t": pd.date_range("20100101", periods=3)} ) grouped = ds.groupby("t.day") - actual = grouped - grouped.mean(ALL_DIMS) + actual = grouped - grouped.mean(...) expected = Dataset({"x": ("t", [0, 0, 0])}, ds[["t", "t.day"]]) assert_identical(actual, expected) def test_groupby_nan(self): # nan should be excluded from groupby ds = Dataset({"foo": ("x", [1, 2, 3, 4])}, {"bar": ("x", [1, 1, 2, np.nan])}) - actual = ds.groupby("bar").mean(ALL_DIMS) + actual = ds.groupby("bar").mean(...) expected = Dataset({"foo": ("bar", [1.5, 3]), "bar": [1, 2]}) assert_identical(actual, expected) @@ -3421,7 +3420,7 @@ def test_groupby_order(self): for vn in ["a", "b", "c"]: ds[vn] = DataArray(np.arange(10), dims=["t"]) data_vars_ref = list(ds.data_vars.keys()) - ds = ds.groupby("t").mean(ALL_DIMS) + ds = ds.groupby("t").mean(...) data_vars = list(ds.data_vars.keys()) assert data_vars == data_vars_ref # coords are now at the end of the list, so the test below fails diff --git a/xarray/tests/test_groupby.py b/xarray/tests/test_groupby.py index be494c4ae2b..a6de41beb66 100644 --- a/xarray/tests/test_groupby.py +++ b/xarray/tests/test_groupby.py @@ -147,11 +147,11 @@ def test_da_groupby_quantile(): [("x", [1, 1, 1, 2, 2]), ("y", [0, 0, 1])], ) - actual_x = array.groupby("x").quantile(0, dim=xr.ALL_DIMS) + actual_x = array.groupby("x").quantile(0, dim=...) expected_x = xr.DataArray([1, 4], [("x", [1, 2])]) assert_identical(expected_x, actual_x) - actual_y = array.groupby("y").quantile(0, dim=xr.ALL_DIMS) + actual_y = array.groupby("y").quantile(0, dim=...) expected_y = xr.DataArray([1, 22], [("y", [0, 1])]) assert_identical(expected_y, actual_y) @@ -177,7 +177,7 @@ def test_da_groupby_quantile(): ) g = foo.groupby(foo.time.dt.month) - actual = g.quantile(0, dim=xr.ALL_DIMS) + actual = g.quantile(0, dim=...) expected = xr.DataArray( [ 0.0, diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 3ac45a9720f..7deabd46eae 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -417,7 +417,7 @@ def test_convenient_facetgrid_4d(self): def test_coord_with_interval(self): bins = [-1, 0, 1, 2] - self.darray.groupby_bins("dim_0", bins).mean(xr.ALL_DIMS).plot() + self.darray.groupby_bins("dim_0", bins).mean(...).plot() class TestPlot1D(PlotTestCase): @@ -502,7 +502,7 @@ def test_step(self): def test_coord_with_interval_step(self): bins = [-1, 0, 1, 2] - self.darray.groupby_bins("dim_0", bins).mean(xr.ALL_DIMS).plot.step() + self.darray.groupby_bins("dim_0", bins).mean(...).plot.step() assert len(plt.gca().lines[0].get_xdata()) == ((len(bins) - 1) * 2) @@ -544,7 +544,7 @@ def test_plot_nans(self): def test_hist_coord_with_interval(self): ( self.darray.groupby_bins("dim_0", [-1, 0, 1, 2]) - .mean(xr.ALL_DIMS) + .mean(...) .plot.hist(range=(-1, 2)) ) diff --git a/xarray/tests/test_sparse.py b/xarray/tests/test_sparse.py index bd26b96f6d4..73c4b9b8c74 100644 --- a/xarray/tests/test_sparse.py +++ b/xarray/tests/test_sparse.py @@ -756,8 +756,8 @@ def test_dot(self): def test_groupby(self): x1 = self.ds_xr x2 = self.sp_xr - m1 = x1.groupby("x").mean(xr.ALL_DIMS) - m2 = x2.groupby("x").mean(xr.ALL_DIMS) + m1 = x1.groupby("x").mean(...) + m2 = x2.groupby("x").mean(...) assert isinstance(m2.data, sparse.SparseArray) assert np.allclose(m1.data, m2.data.todense()) @@ -772,8 +772,8 @@ def test_groupby_first(self): def test_groupby_bins(self): x1 = self.ds_xr x2 = self.sp_xr - m1 = x1.groupby_bins("x", bins=[0, 3, 7, 10]).sum(xr.ALL_DIMS) - m2 = x2.groupby_bins("x", bins=[0, 3, 7, 10]).sum(xr.ALL_DIMS) + m1 = x1.groupby_bins("x", bins=[0, 3, 7, 10]).sum(...) + m2 = x2.groupby_bins("x", bins=[0, 3, 7, 10]).sum(...) assert isinstance(m2.data, sparse.SparseArray) assert np.allclose(m1.data, m2.data.todense()) From 63cc85759ac25605c8398d904d055df5dc538b94 Mon Sep 17 00:00:00 2001 From: Benoit Bovy Date: Fri, 25 Oct 2019 17:40:46 +0200 Subject: [PATCH 4/5] add icomoon license (#3448) --- README.rst | 3 + licenses/ICOMOON_LICENSE | 395 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 398 insertions(+) create mode 100644 licenses/ICOMOON_LICENSE diff --git a/README.rst b/README.rst index 53f51392a1a..5ee7234f221 100644 --- a/README.rst +++ b/README.rst @@ -138,4 +138,7 @@ under a "3-clause BSD" license: xarray also bundles portions of CPython, which is available under the "Python Software Foundation License" in xarray/core/pycompat.py. +xarray uses icons from the icomoon package (free version), which is +available under the "CC BY 4.0" license. + The full text of these licenses are included in the licenses directory. diff --git a/licenses/ICOMOON_LICENSE b/licenses/ICOMOON_LICENSE new file mode 100644 index 00000000000..4ea99c213c5 --- /dev/null +++ b/licenses/ICOMOON_LICENSE @@ -0,0 +1,395 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. From fb0cf7b5fe56519a933ffcecbce9e9327fe236a6 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 25 Oct 2019 15:01:11 -0600 Subject: [PATCH 5/5] Another groupby.reduce bugfix. (#3403) * Another groupby.reduce bugfix. Fixes #3402 * Add whats-new. * Use is_scalar instead * bugfix * fix whats-new * Update xarray/core/groupby.py Co-Authored-By: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> --- doc/whats-new.rst | 4 +++ xarray/core/groupby.py | 27 +++++++++------- xarray/tests/test_dataarray.py | 9 ------ xarray/tests/test_groupby.py | 56 +++++++++++++++++++++++++--------- 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ac60994d35b..dea110b5e46 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -55,6 +55,10 @@ Bug fixes - Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4. By `Anderson Banihirwe `_. +- Fix :py:meth:`xarray.core.groupby.DataArrayGroupBy.reduce` and + :py:meth:`xarray.core.groupby.DatasetGroupBy.reduce` when reducing over multiple dimensions. + (:issue:`3402`). By `Deepak Cherian `_ + Documentation ~~~~~~~~~~~~~ diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index 68bd28ddb12..62c055fed51 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -15,6 +15,7 @@ from .utils import ( either_dict_or_kwargs, hashable, + is_scalar, maybe_wrap_array, peek_at, safe_cast_to_index, @@ -22,6 +23,18 @@ from .variable import IndexVariable, Variable, as_variable +def check_reduce_dims(reduce_dims, dimensions): + + if reduce_dims is not ...: + if is_scalar(reduce_dims): + reduce_dims = [reduce_dims] + if any([dim not in dimensions for dim in reduce_dims]): + raise ValueError( + "cannot reduce over dimensions %r. expected either '...' to reduce over all dimensions or one or more of %r." + % (reduce_dims, dimensions) + ) + + def unique_value_groups(ar, sort=True): """Group an array by its unique values. @@ -794,15 +807,11 @@ def reduce( if keep_attrs is None: keep_attrs = _get_keep_attrs(default=False) - if dim is not ... and dim not in self.dims: - raise ValueError( - "cannot reduce over dimension %r. expected either '...' to reduce over all dimensions or one or more of %r." - % (dim, self.dims) - ) - def reduce_array(ar): return ar.reduce(func, dim, axis, keep_attrs=keep_attrs, **kwargs) + check_reduce_dims(dim, self.dims) + return self.apply(reduce_array, shortcut=shortcut) @@ -895,11 +904,7 @@ def reduce(self, func, dim=None, keep_attrs=None, **kwargs): def reduce_dataset(ds): return ds.reduce(func, dim, keep_attrs, **kwargs) - if dim is not ... and dim not in self.dims: - raise ValueError( - "cannot reduce over dimension %r. expected either '...' to reduce over all dimensions or one or more of %r." - % (dim, self.dims) - ) + check_reduce_dims(dim, self.dims) return self.apply(reduce_dataset) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index b13527bc098..101bb44660c 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2560,15 +2560,6 @@ def change_metadata(x): expected = change_metadata(expected) assert_equal(expected, actual) - def test_groupby_reduce_dimension_error(self): - array = self.make_groupby_example_array() - grouped = array.groupby("y") - with raises_regex(ValueError, "cannot reduce over dimension 'y'"): - grouped.mean() - - grouped = array.groupby("y", squeeze=False) - assert_identical(array, grouped.mean()) - def test_groupby_math(self): array = self.make_groupby_example_array() for squeeze in [True, False]: diff --git a/xarray/tests/test_groupby.py b/xarray/tests/test_groupby.py index a6de41beb66..d74d684dc54 100644 --- a/xarray/tests/test_groupby.py +++ b/xarray/tests/test_groupby.py @@ -5,7 +5,23 @@ import xarray as xr from xarray.core.groupby import _consolidate_slices -from . import assert_identical, raises_regex +from . import assert_allclose, assert_identical, raises_regex + + +@pytest.fixture +def dataset(): + ds = xr.Dataset( + {"foo": (("x", "y", "z"), np.random.randn(3, 4, 2))}, + {"x": ["a", "b", "c"], "y": [1, 2, 3, 4], "z": [1, 2]}, + ) + ds["boo"] = (("z", "y"), [["f", "g", "h", "j"]] * 2) + + return ds + + +@pytest.fixture +def array(dataset): + return dataset["foo"] def test_consolidate_slices(): @@ -21,25 +37,17 @@ def test_consolidate_slices(): _consolidate_slices([slice(3), 4]) -def test_groupby_dims_property(): - ds = xr.Dataset( - {"foo": (("x", "y", "z"), np.random.randn(3, 4, 2))}, - {"x": ["a", "bcd", "c"], "y": [1, 2, 3, 4], "z": [1, 2]}, - ) +def test_groupby_dims_property(dataset): + assert dataset.groupby("x").dims == dataset.isel(x=1).dims + assert dataset.groupby("y").dims == dataset.isel(y=1).dims - assert ds.groupby("x").dims == ds.isel(x=1).dims - assert ds.groupby("y").dims == ds.isel(y=1).dims - - stacked = ds.stack({"xy": ("x", "y")}) + stacked = dataset.stack({"xy": ("x", "y")}) assert stacked.groupby("xy").dims == stacked.isel(xy=0).dims -def test_multi_index_groupby_apply(): +def test_multi_index_groupby_apply(dataset): # regression test for GH873 - ds = xr.Dataset( - {"foo": (("x", "y"), np.random.randn(3, 4))}, - {"x": ["a", "b", "c"], "y": [1, 2, 3, 4]}, - ) + ds = dataset.isel(z=1, drop=True)[["foo"]] doubled = 2 * ds group_doubled = ( ds.stack(space=["x", "y"]) @@ -276,6 +284,24 @@ def test_groupby_grouping_errors(): dataset.to_array().groupby(dataset.foo * np.nan) +def test_groupby_reduce_dimension_error(array): + grouped = array.groupby("y") + with raises_regex(ValueError, "cannot reduce over dimensions"): + grouped.mean() + + with raises_regex(ValueError, "cannot reduce over dimensions"): + grouped.mean("huh") + + with raises_regex(ValueError, "cannot reduce over dimensions"): + grouped.mean(("x", "y", "asd")) + + grouped = array.groupby("y", squeeze=False) + assert_identical(array, grouped.mean()) + + assert_identical(array.mean("x"), grouped.reduce(np.mean, "x")) + assert_allclose(array.mean(["x", "z"]), grouped.reduce(np.mean, ["x", "z"])) + + def test_groupby_bins_timeseries(): ds = xr.Dataset() ds["time"] = xr.DataArray(