Skip to content

Commit

Permalink
"blockwise" need not be the best strategy for resample..
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcherian
dcherian committed Nov 15, 2021
1 parent c189eea commit 43ade8c
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 66 deletions.
90 changes: 30 additions & 60 deletions xarray/core/_reductions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2057,8 +2057,7 @@ def count(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2146,8 +2145,7 @@ def all(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2235,8 +2233,7 @@ def any(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2341,8 +2338,7 @@ def max(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2448,8 +2444,7 @@ def min(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2559,8 +2554,7 @@ def mean(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2688,8 +2682,7 @@ def prod(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2818,8 +2811,7 @@ def sum(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -2945,8 +2937,7 @@ def std(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3072,8 +3063,7 @@ def var(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3261,8 +3251,7 @@ def count(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3350,8 +3339,7 @@ def all(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3439,8 +3427,7 @@ def any(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3545,8 +3532,7 @@ def max(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3652,8 +3638,7 @@ def min(
numeric_only=False,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3763,8 +3748,7 @@ def mean(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -3892,8 +3876,7 @@ def prod(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4022,8 +4005,7 @@ def sum(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4149,8 +4131,7 @@ def std(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4276,8 +4257,7 @@ def var(
numeric_only=True,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4459,8 +4439,7 @@ def count(
dim=dim,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4541,8 +4520,7 @@ def all(
dim=dim,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4623,8 +4601,7 @@ def any(
dim=dim,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4720,8 +4697,7 @@ def max(
skipna=skipna,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4818,8 +4794,7 @@ def min(
skipna=skipna,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -4920,8 +4895,7 @@ def mean(
skipna=skipna,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -5038,8 +5012,7 @@ def prod(
min_count=min_count,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -5157,8 +5130,7 @@ def sum(
min_count=min_count,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -5273,8 +5245,7 @@ def std(
ddof=ddof,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down Expand Up @@ -5389,8 +5360,7 @@ def var(
ddof=ddof,
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down
2 changes: 0 additions & 2 deletions xarray/core/groupby.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,6 @@ class GroupBy:
"_stacked_dim",
"_unique_coord",
"_dims",
"_dask_groupby_kwargs",
"_squeeze",
# Save unstacked object for dask_groupby
"_original_obj",
Expand Down Expand Up @@ -406,7 +405,6 @@ def __init__(
self._inserted_dims = inserted_dims
self._full_index = full_index
self._restore_coord_dims = restore_coord_dims
self._dask_groupby_kwargs = {}
self._squeeze = squeeze
# self._by = by

Expand Down
2 changes: 0 additions & 2 deletions xarray/core/resample.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,6 @@ def __init__(self, *args, dim=None, resample_dim=None, **kwargs):

super().__init__(*args, **kwargs)

self._dask_groupby_kwargs = dict(method="blockwise")

def map(self, func, shortcut=False, args=(), **kwargs):
"""Apply a function to each array in the group and concatenate them
together into a new array.
Expand Down
3 changes: 1 addition & 2 deletions xarray/util/generate_reductions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,8 +363,7 @@ def generate_code(self, method):
dim=dim,{extra_kwargs}
# fill_value=fill_value,
keep_attrs=keep_attrs,
# TODO: Add dask resampling reduction tests!
**self._dask_groupby_kwargs,
**kwargs,
)
else:
return self.reduce(
Expand Down

0 comments on commit 43ade8c

Please sign in to comment.