From fdd43c4de98992a6f97a835fdbb525f829ef1d69 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 1 Oct 2018 21:26:09 -0500 Subject: [PATCH 01/12] Closes https://github.com/pandas-dev/pandas/issues/22850 --- pandas/core/series.py | 4 +++- pandas/tests/extension/test_ops.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 pandas/tests/extension/test_ops.py diff --git a/pandas/core/series.py b/pandas/core/series.py index 83f80c305c5eb..7c1192374fc44 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -2324,9 +2324,11 @@ def combine(self, other, func, fill_value=None): elif is_extension_array_dtype(self.values): # The function can return something of any type, so check # if the type is compatible with the calling EA + # ExtensionArray._from_sequence can raise anything, so we + # have to catch everything. try: new_values = self._values._from_sequence(new_values) - except TypeError: + except Exception: pass return self._constructor(new_values, index=new_index, name=new_name) diff --git a/pandas/tests/extension/test_ops.py b/pandas/tests/extension/test_ops.py new file mode 100644 index 0000000000000..5d23fc8bf899a --- /dev/null +++ b/pandas/tests/extension/test_ops.py @@ -0,0 +1,20 @@ +import operator +from decimal import Decimal + +import pandas as pd +import pandas.util.testing as tm +from pandas.tests.extension.decimal.array import DecimalArray + + +def test_combine_from_sequence_raises(): + # https://github.com/pandas-dev/pandas/issues/22850 + class BadDecimalArray(DecimalArray): + def _from_sequence(cls, scalars, dtype=None, copy=False): + raise KeyError("For the test") + + ser = pd.Series(BadDecimalArray([Decimal("1.0"), Decimal("2.0")])) + result = ser.combine(ser, operator.add) + + # note: object dtype + expected = pd.Series([Decimal("2.0"), Decimal("4.0")], dtype="object") + tm.assert_series_equal(result, expected) From ce94bf9b9c6799bd4220976022895edd4e60abd5 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 06:28:16 -0500 Subject: [PATCH 02/12] Moves --- pandas/tests/extension/decimal_array/__init__.py | 1 + pandas/tests/extension/{decimal => decimal_array}/array.py | 0 .../tests/extension/{decimal => decimal_array}/test_decimal.py | 0 pandas/tests/extension/json/__init__.py | 0 pandas/tests/extension/{decimal => json_array}/__init__.py | 0 pandas/tests/extension/{json => json_array}/array.py | 0 pandas/tests/extension/{json => json_array}/test_json.py | 0 pandas/tests/extension/test_ops.py | 2 +- 8 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 pandas/tests/extension/decimal_array/__init__.py rename pandas/tests/extension/{decimal => decimal_array}/array.py (100%) rename pandas/tests/extension/{decimal => decimal_array}/test_decimal.py (100%) delete mode 100644 pandas/tests/extension/json/__init__.py rename pandas/tests/extension/{decimal => json_array}/__init__.py (100%) rename pandas/tests/extension/{json => json_array}/array.py (100%) rename pandas/tests/extension/{json => json_array}/test_json.py (100%) diff --git a/pandas/tests/extension/decimal_array/__init__.py b/pandas/tests/extension/decimal_array/__init__.py new file mode 100644 index 0000000000000..05bb5f95b0fa6 --- /dev/null +++ b/pandas/tests/extension/decimal_array/__init__.py @@ -0,0 +1 @@ +from .array import DecimalArray, DecimalDtype \ No newline at end of file diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal_array/array.py similarity index 100% rename from pandas/tests/extension/decimal/array.py rename to pandas/tests/extension/decimal_array/array.py diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal_array/test_decimal.py similarity index 100% rename from pandas/tests/extension/decimal/test_decimal.py rename to pandas/tests/extension/decimal_array/test_decimal.py diff --git a/pandas/tests/extension/json/__init__.py b/pandas/tests/extension/json/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/pandas/tests/extension/decimal/__init__.py b/pandas/tests/extension/json_array/__init__.py similarity index 100% rename from pandas/tests/extension/decimal/__init__.py rename to pandas/tests/extension/json_array/__init__.py diff --git a/pandas/tests/extension/json/array.py b/pandas/tests/extension/json_array/array.py similarity index 100% rename from pandas/tests/extension/json/array.py rename to pandas/tests/extension/json_array/array.py diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json_array/test_json.py similarity index 100% rename from pandas/tests/extension/json/test_json.py rename to pandas/tests/extension/json_array/test_json.py diff --git a/pandas/tests/extension/test_ops.py b/pandas/tests/extension/test_ops.py index 5d23fc8bf899a..bf2c23b58b6b6 100644 --- a/pandas/tests/extension/test_ops.py +++ b/pandas/tests/extension/test_ops.py @@ -3,7 +3,7 @@ import pandas as pd import pandas.util.testing as tm -from pandas.tests.extension.decimal.array import DecimalArray +from pandas.tests.extension.decimal_array import DecimalArray def test_combine_from_sequence_raises(): From 0c53f080b419151286b6991acff540150f13fccc Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 06:30:54 -0500 Subject: [PATCH 03/12] Imports --- pandas/tests/extension/decimal_array/__init__.py | 5 ++++- pandas/tests/extension/json_array/__init__.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/tests/extension/decimal_array/__init__.py b/pandas/tests/extension/decimal_array/__init__.py index 05bb5f95b0fa6..9a0071e1c776e 100644 --- a/pandas/tests/extension/decimal_array/__init__.py +++ b/pandas/tests/extension/decimal_array/__init__.py @@ -1 +1,4 @@ -from .array import DecimalArray, DecimalDtype \ No newline at end of file +from .array import DecimalArray, DecimalDtype + + +__all__ = ['DecimalArray', 'DecimalDtype'] diff --git a/pandas/tests/extension/json_array/__init__.py b/pandas/tests/extension/json_array/__init__.py index e69de29bb2d1d..a314a9c10d3dd 100644 --- a/pandas/tests/extension/json_array/__init__.py +++ b/pandas/tests/extension/json_array/__init__.py @@ -0,0 +1,3 @@ +from .array import JSONArray, JSONDtype + +__all__ = ['JSONArray', 'JSONDtype'] From 9059c0d23357a717f5f7ba8f5e165000b70efcc2 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 06:33:15 -0500 Subject: [PATCH 04/12] Note --- doc/source/whatsnew/v0.24.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index b71edcf1f6f51..eb9e6f5b9981a 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -505,6 +505,7 @@ ExtensionType Changes - :meth:`Series.astype` and :meth:`DataFrame.astype` now dispatch to :meth:`ExtensionArray.astype` (:issue:`21185:`). - Slicing a single row of a ``DataFrame`` with multiple ExtensionArrays of the same type now preserves the dtype, rather than coercing to object (:issue:`22784`) - Added :meth:`pandas.api.types.register_extension_dtype` to register an extension type with pandas (:issue:`22664`) +- Renamed test prototype module names to avoid conflict with modules from the standard library (:issue:`22936`) .. _whatsnew_0240.api.incompatibilities: From 2183f7bf71a90661877c87fe04dcd52efa481184 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:17:28 -0500 Subject: [PATCH 05/12] api --- pandas/tests/extension/decimal_array/__init__.py | 2 +- pandas/tests/extension/decimal_array/array.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/tests/extension/decimal_array/__init__.py b/pandas/tests/extension/decimal_array/__init__.py index 9a0071e1c776e..6bdfa0e05e0a0 100644 --- a/pandas/tests/extension/decimal_array/__init__.py +++ b/pandas/tests/extension/decimal_array/__init__.py @@ -1,4 +1,4 @@ from .array import DecimalArray, DecimalDtype -__all__ = ['DecimalArray', 'DecimalDtype'] +__all__ = ['DecimalArray', 'DecimalDtype', 'to_decimal', 'make_data'] diff --git a/pandas/tests/extension/decimal_array/array.py b/pandas/tests/extension/decimal_array/array.py index 387942234e6fd..f324cc2e0f345 100644 --- a/pandas/tests/extension/decimal_array/array.py +++ b/pandas/tests/extension/decimal_array/array.py @@ -138,5 +138,9 @@ def _concat_same_type(cls, to_concat): return cls(np.concatenate([x._data for x in to_concat])) +def to_decimal(values, context=None): + return DecimalArray([decimal.Decimal(x) for x in values], context=context) + + DecimalArray._add_arithmetic_ops() DecimalArray._add_comparison_ops() From 0eef0cfcdb17caed9cdddce03cb5d07924225375 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:18:18 -0500 Subject: [PATCH 06/12] move back --- pandas/tests/extension/{ => decimal}/decimal_array/__init__.py | 0 pandas/tests/extension/{ => decimal}/decimal_array/array.py | 0 .../tests/extension/{ => decimal}/decimal_array/test_decimal.py | 0 pandas/tests/extension/{json_array => json}/__init__.py | 0 pandas/tests/extension/{json_array => json}/array.py | 0 pandas/tests/extension/{json_array => json}/test_json.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename pandas/tests/extension/{ => decimal}/decimal_array/__init__.py (100%) rename pandas/tests/extension/{ => decimal}/decimal_array/array.py (100%) rename pandas/tests/extension/{ => decimal}/decimal_array/test_decimal.py (100%) rename pandas/tests/extension/{json_array => json}/__init__.py (100%) rename pandas/tests/extension/{json_array => json}/array.py (100%) rename pandas/tests/extension/{json_array => json}/test_json.py (100%) diff --git a/pandas/tests/extension/decimal_array/__init__.py b/pandas/tests/extension/decimal/decimal_array/__init__.py similarity index 100% rename from pandas/tests/extension/decimal_array/__init__.py rename to pandas/tests/extension/decimal/decimal_array/__init__.py diff --git a/pandas/tests/extension/decimal_array/array.py b/pandas/tests/extension/decimal/decimal_array/array.py similarity index 100% rename from pandas/tests/extension/decimal_array/array.py rename to pandas/tests/extension/decimal/decimal_array/array.py diff --git a/pandas/tests/extension/decimal_array/test_decimal.py b/pandas/tests/extension/decimal/decimal_array/test_decimal.py similarity index 100% rename from pandas/tests/extension/decimal_array/test_decimal.py rename to pandas/tests/extension/decimal/decimal_array/test_decimal.py diff --git a/pandas/tests/extension/json_array/__init__.py b/pandas/tests/extension/json/__init__.py similarity index 100% rename from pandas/tests/extension/json_array/__init__.py rename to pandas/tests/extension/json/__init__.py diff --git a/pandas/tests/extension/json_array/array.py b/pandas/tests/extension/json/array.py similarity index 100% rename from pandas/tests/extension/json_array/array.py rename to pandas/tests/extension/json/array.py diff --git a/pandas/tests/extension/json_array/test_json.py b/pandas/tests/extension/json/test_json.py similarity index 100% rename from pandas/tests/extension/json_array/test_json.py rename to pandas/tests/extension/json/test_json.py From be63feb818b1e39d0f711d55c1eac809a03ee061 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:19:17 -0500 Subject: [PATCH 07/12] move test --- .../decimal/decimal_array/test_decimal.py | 17 ++++++++++++++++ pandas/tests/extension/test_ops.py | 20 ------------------- 2 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 pandas/tests/extension/test_ops.py diff --git a/pandas/tests/extension/decimal/decimal_array/test_decimal.py b/pandas/tests/extension/decimal/decimal_array/test_decimal.py index 93b8ea786ef5b..c7a39ce3ff9f5 100644 --- a/pandas/tests/extension/decimal/decimal_array/test_decimal.py +++ b/pandas/tests/extension/decimal/decimal_array/test_decimal.py @@ -1,3 +1,4 @@ +import operator import decimal import random @@ -275,3 +276,19 @@ def test_compare_array(self, data, all_compare_operators): other = pd.Series(data) * [decimal.Decimal(pow(2.0, i)) for i in alter] self._compare_other(s, data, op_name, other) + + +def test_combine_from_sequence_raises(): + # https://github.com/pandas-dev/pandas/issues/22850 + class BadDecimalArray(DecimalArray): + def _from_sequence(cls, scalars, dtype=None, copy=False): + raise KeyError("For the test") + + ser = pd.Series(BadDecimalArray([decimal.Decimal("1.0"), + decimal.Decimal("2.0")])) + result = ser.combine(ser, operator.add) + + # note: object dtype + expected = pd.Series([decimal.Decimal("2.0"), + decimal.Decimal("4.0")], dtype="object") + tm.assert_series_equal(result, expected) diff --git a/pandas/tests/extension/test_ops.py b/pandas/tests/extension/test_ops.py deleted file mode 100644 index bf2c23b58b6b6..0000000000000 --- a/pandas/tests/extension/test_ops.py +++ /dev/null @@ -1,20 +0,0 @@ -import operator -from decimal import Decimal - -import pandas as pd -import pandas.util.testing as tm -from pandas.tests.extension.decimal_array import DecimalArray - - -def test_combine_from_sequence_raises(): - # https://github.com/pandas-dev/pandas/issues/22850 - class BadDecimalArray(DecimalArray): - def _from_sequence(cls, scalars, dtype=None, copy=False): - raise KeyError("For the test") - - ser = pd.Series(BadDecimalArray([Decimal("1.0"), Decimal("2.0")])) - result = ser.combine(ser, operator.add) - - # note: object dtype - expected = pd.Series([Decimal("2.0"), Decimal("4.0")], dtype="object") - tm.assert_series_equal(result, expected) From a4a2933117c493394a5656c887a22ea02e94093f Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:24:48 -0500 Subject: [PATCH 08/12] handle test --- pandas/core/arrays/base.py | 4 +-- .../decimal/decimal_array/test_decimal.py | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 7bf13fb2fecc0..9f515c38c14ce 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -777,8 +777,8 @@ def convert_values(param): if coerce_to_dtype: try: res = self._from_sequence(res) - except TypeError: - pass + except Exception: + res = np.asarray(res, dtype=object) return res diff --git a/pandas/tests/extension/decimal/decimal_array/test_decimal.py b/pandas/tests/extension/decimal/decimal_array/test_decimal.py index c7a39ce3ff9f5..024f758943649 100644 --- a/pandas/tests/extension/decimal/decimal_array/test_decimal.py +++ b/pandas/tests/extension/decimal/decimal_array/test_decimal.py @@ -278,17 +278,32 @@ def test_compare_array(self, data, all_compare_operators): self._compare_other(s, data, op_name, other) +class DecimalArrayWithoutFromSequence(DecimalArray): + """Helper class for testing error handling in _from_sequence.""" + def _from_sequence(cls, scalars, dtype=None, copy=False): + raise KeyError("For the test") + + def test_combine_from_sequence_raises(): # https://github.com/pandas-dev/pandas/issues/22850 - class BadDecimalArray(DecimalArray): - def _from_sequence(cls, scalars, dtype=None, copy=False): - raise KeyError("For the test") - - ser = pd.Series(BadDecimalArray([decimal.Decimal("1.0"), - decimal.Decimal("2.0")])) + ser = pd.Series(DecimalArrayWithoutFromSequence([ + decimal.Decimal("1.0"), + decimal.Decimal("2.0") + ])) result = ser.combine(ser, operator.add) # note: object dtype expected = pd.Series([decimal.Decimal("2.0"), decimal.Decimal("4.0")], dtype="object") tm.assert_series_equal(result, expected) + + +def test_scalar_ops_from_sequence_raises(): + arr = DecimalArrayWithoutFromSequence([ + decimal.Decimal("1.0"), + decimal.Decimal("2.0") + ]) + result = arr + arr + expected = np.array([decimal.Decimal("2.0"), decimal.Decimal("4.0")], + dtype="object") + tm.assert_numpy_array_equal(result, expected) From b9c7e4b2c0577fed6601e4f6e27974b943f280da Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:28:57 -0500 Subject: [PATCH 09/12] remove old note --- doc/source/whatsnew/v0.24.0.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 84ef226fa9821..851c1a3fbd6e9 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -505,7 +505,6 @@ ExtensionType Changes - :meth:`Series.astype` and :meth:`DataFrame.astype` now dispatch to :meth:`ExtensionArray.astype` (:issue:`21185:`). - Slicing a single row of a ``DataFrame`` with multiple ExtensionArrays of the same type now preserves the dtype, rather than coercing to object (:issue:`22784`) - Added :meth:`pandas.api.types.register_extension_dtype` to register an extension type with pandas (:issue:`22664`) -- Renamed test prototype module names to avoid conflict with modules from the standard library (:issue:`22936`) .. _whatsnew_0240.api.incompatibilities: From ce1a3c6b112a6228847f9c622ef8246c671f7170 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:32:11 -0500 Subject: [PATCH 10/12] fixed move --- .../tests/extension/decimal/{decimal_array => }/__init__.py | 0 pandas/tests/extension/decimal/{decimal_array => }/array.py | 0 .../extension/decimal/{decimal_array => }/test_decimal.py | 0 pandas/tests/extension/json/__init__.py | 4 ++-- 4 files changed, 2 insertions(+), 2 deletions(-) rename pandas/tests/extension/decimal/{decimal_array => }/__init__.py (100%) rename pandas/tests/extension/decimal/{decimal_array => }/array.py (100%) rename pandas/tests/extension/decimal/{decimal_array => }/test_decimal.py (100%) diff --git a/pandas/tests/extension/decimal/decimal_array/__init__.py b/pandas/tests/extension/decimal/__init__.py similarity index 100% rename from pandas/tests/extension/decimal/decimal_array/__init__.py rename to pandas/tests/extension/decimal/__init__.py diff --git a/pandas/tests/extension/decimal/decimal_array/array.py b/pandas/tests/extension/decimal/array.py similarity index 100% rename from pandas/tests/extension/decimal/decimal_array/array.py rename to pandas/tests/extension/decimal/array.py diff --git a/pandas/tests/extension/decimal/decimal_array/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py similarity index 100% rename from pandas/tests/extension/decimal/decimal_array/test_decimal.py rename to pandas/tests/extension/decimal/test_decimal.py diff --git a/pandas/tests/extension/json/__init__.py b/pandas/tests/extension/json/__init__.py index a314a9c10d3dd..f2679d087c841 100644 --- a/pandas/tests/extension/json/__init__.py +++ b/pandas/tests/extension/json/__init__.py @@ -1,3 +1,3 @@ -from .array import JSONArray, JSONDtype +from .array import JSONArray, JSONDtype, make_data -__all__ = ['JSONArray', 'JSONDtype'] +__all__ = ['JSONArray', 'JSONDtype', 'make_data'] From 5372134ea2d22c90fff4b5830464a8f2c9932407 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Oct 2018 11:35:07 -0500 Subject: [PATCH 11/12] fixed move --- pandas/tests/extension/decimal/__init__.py | 2 +- pandas/tests/extension/decimal/array.py | 5 +++++ pandas/tests/extension/decimal/test_decimal.py | 7 +------ pandas/tests/extension/json/array.py | 9 +++++++++ pandas/tests/extension/json/test_json.py | 11 +---------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pandas/tests/extension/decimal/__init__.py b/pandas/tests/extension/decimal/__init__.py index 6bdfa0e05e0a0..c37aad0af8407 100644 --- a/pandas/tests/extension/decimal/__init__.py +++ b/pandas/tests/extension/decimal/__init__.py @@ -1,4 +1,4 @@ -from .array import DecimalArray, DecimalDtype +from .array import DecimalArray, DecimalDtype, to_decimal, make_data __all__ = ['DecimalArray', 'DecimalDtype', 'to_decimal', 'make_data'] diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index f324cc2e0f345..79e1a692f744a 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -1,5 +1,6 @@ import decimal import numbers +import random import sys import numpy as np @@ -142,5 +143,9 @@ def to_decimal(values, context=None): return DecimalArray([decimal.Decimal(x) for x in values], context=context) +def make_data(): + return [decimal.Decimal(random.random()) for _ in range(100)] + + DecimalArray._add_arithmetic_ops() DecimalArray._add_comparison_ops() diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py index 024f758943649..64872eba6f8f0 100644 --- a/pandas/tests/extension/decimal/test_decimal.py +++ b/pandas/tests/extension/decimal/test_decimal.py @@ -1,7 +1,6 @@ import operator import decimal -import random import numpy as np import pandas as pd import pandas.util.testing as tm @@ -9,11 +8,7 @@ from pandas.tests.extension import base -from .array import DecimalDtype, DecimalArray - - -def make_data(): - return [decimal.Decimal(random.random()) for _ in range(100)] +from .array import DecimalDtype, DecimalArray, make_data @pytest.fixture diff --git a/pandas/tests/extension/json/array.py b/pandas/tests/extension/json/array.py index 6ce0d63eb63ec..87876d84bef99 100644 --- a/pandas/tests/extension/json/array.py +++ b/pandas/tests/extension/json/array.py @@ -13,6 +13,8 @@ import collections import itertools import numbers +import random +import string import sys import numpy as np @@ -179,3 +181,10 @@ def _values_for_argsort(self): # cast them to an (N, P) array, instead of an (N,) array of tuples. frozen = [()] + [tuple(x.items()) for x in self] return np.array(frozen, dtype=object)[1:] + + +def make_data(): + # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer + return [collections.UserDict([ + (random.choice(string.ascii_letters), random.randint(0, 100)) + for _ in range(random.randint(0, 10))]) for _ in range(100)] diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py index 0126d771caf7f..b30ff02c4fa9a 100644 --- a/pandas/tests/extension/json/test_json.py +++ b/pandas/tests/extension/json/test_json.py @@ -1,7 +1,5 @@ import operator import collections -import random -import string import pytest @@ -10,18 +8,11 @@ from pandas.compat import PY2, PY36 from pandas.tests.extension import base -from .array import JSONArray, JSONDtype +from .array import JSONArray, JSONDtype, make_data pytestmark = pytest.mark.skipif(PY2, reason="Py2 doesn't have a UserDict") -def make_data(): - # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer - return [collections.UserDict([ - (random.choice(string.ascii_letters), random.randint(0, 100)) - for _ in range(random.randint(0, 10))]) for _ in range(100)] - - @pytest.fixture def dtype(): return JSONDtype() From 7714e79a512f5d13f52c286458fcc9bac8b40bd0 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 3 Oct 2018 10:13:06 -0500 Subject: [PATCH 12/12] Always return ndarray --- pandas/core/arrays/base.py | 24 ++++++++++++++----- pandas/core/series.py | 8 ++++--- .../tests/extension/decimal/test_decimal.py | 17 +++++++++++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 9f515c38c14ce..f7c4ee35adfe4 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -739,14 +739,22 @@ def _create_method(cls, op, coerce_to_dtype=True): ---------- op : function An operator that takes arguments op(a, b) - coerce_to_dtype : bool + coerce_to_dtype : bool, default True boolean indicating whether to attempt to convert - the result to the underlying ExtensionArray dtype - (default True) + the result to the underlying ExtensionArray dtype. + If it's not possible to create a new ExtensionArray with the + values, an ndarray is returned instead. Returns ------- - A method that can be bound to a method of a class + Callable[[Any, Any], Union[ndarray, ExtensionArray]] + A method that can be bound to a class. When used, the method + receives the two arguments, one of which is the instance of + this class, and should return an ExtensionArray or an ndarray. + + Returning an ndarray may be necessary when the result of the + `op` cannot be stored in the ExtensionArray. The dtype of the + ndarray uses NumPy's normal inference rules. Example ------- @@ -757,7 +765,6 @@ def _create_method(cls, op, coerce_to_dtype=True): in the class definition of MyExtensionArray to create the operator for addition, that will be based on the operator implementation of the underlying elements of the ExtensionArray - """ def _binop(self, other): @@ -778,7 +785,12 @@ def convert_values(param): try: res = self._from_sequence(res) except Exception: - res = np.asarray(res, dtype=object) + # https://github.com/pandas-dev/pandas/issues/22850 + # We catch all regular exceptions here, and fall back + # to an ndarray. + res = np.asarray(res) + else: + res = np.asarray(res) return res diff --git a/pandas/core/series.py b/pandas/core/series.py index 6eb0ce362c497..2e22e4e6e1bfc 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -2323,12 +2323,14 @@ def combine(self, other, func, fill_value=None): pass elif is_extension_array_dtype(self.values): # The function can return something of any type, so check - # if the type is compatible with the calling EA - # ExtensionArray._from_sequence can raise anything, so we - # have to catch everything. + # if the type is compatible with the calling EA. try: new_values = self._values._from_sequence(new_values) except Exception: + # https://github.com/pandas-dev/pandas/issues/22850 + # pandas has no control over what 3rd-party ExtensionArrays + # do in _values_from_sequence. We still want ops to work + # though, so we catch any regular Exception. pass return self._constructor(new_values, index=new_index, name=new_name) diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py index 64872eba6f8f0..dd625d6e1eb3c 100644 --- a/pandas/tests/extension/decimal/test_decimal.py +++ b/pandas/tests/extension/decimal/test_decimal.py @@ -279,6 +279,15 @@ def _from_sequence(cls, scalars, dtype=None, copy=False): raise KeyError("For the test") +class DecimalArrayWithoutCoercion(DecimalArrayWithoutFromSequence): + @classmethod + def _create_arithmetic_method(cls, op): + return cls._create_method(op, coerce_to_dtype=False) + + +DecimalArrayWithoutCoercion._add_arithmetic_ops() + + def test_combine_from_sequence_raises(): # https://github.com/pandas-dev/pandas/issues/22850 ser = pd.Series(DecimalArrayWithoutFromSequence([ @@ -293,8 +302,12 @@ def test_combine_from_sequence_raises(): tm.assert_series_equal(result, expected) -def test_scalar_ops_from_sequence_raises(): - arr = DecimalArrayWithoutFromSequence([ +@pytest.mark.parametrize("class_", [DecimalArrayWithoutFromSequence, + DecimalArrayWithoutCoercion]) +def test_scalar_ops_from_sequence_raises(class_): + # op(EA, EA) should return an EA, or an ndarray if it's not possible + # to return an EA with the return values. + arr = class_([ decimal.Decimal("1.0"), decimal.Decimal("2.0") ])