From ee426abbb4aa480a55076d938a8bfb3ba2dca3cf Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 09:36:28 -0700 Subject: [PATCH 01/11] BUG: fix+test setitem datetime64(NaT) into Series[timedelta64] --- pandas/_libs/index.pyx | 4 ++- pandas/core/internals/blocks.py | 3 +- pandas/core/series.py | 3 +- pandas/tests/series/indexing/test_indexing.py | 35 +++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx index ba2838d59f814..fc543c0bfa322 100644 --- a/pandas/_libs/index.pyx +++ b/pandas/_libs/index.pyx @@ -545,7 +545,9 @@ cpdef convert_scalar(ndarray arr, object value): elif isinstance(value, timedelta): return Timedelta(value).value elif value is None or value != value: - return NPY_NAT + if not util.is_datetime64_object(value): + # exclude np.datetime64("NaT") + return NPY_NAT elif isinstance(value, str): return Timedelta(value).value raise ValueError("cannot set a Timedelta with a non-timedelta") diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index bf6ebf1abe760..7e42df2cfb0f3 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2665,7 +2665,8 @@ def _try_coerce_args(self, other): base-type other """ - if is_null_datetimelike(other): + if is_null_datetimelike(other) and not isinstance(other, np.datetime64): + # exclude np.datetime64("NaT") other = tslibs.iNaT elif isinstance(other, (timedelta, np.timedelta64)): other = Timedelta(other).value diff --git a/pandas/core/series.py b/pandas/core/series.py index b3a7f38aef8ef..356dfe4b6dcb4 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1210,7 +1210,8 @@ def setitem(key, value): pass elif is_timedelta64_dtype(self.dtype): # reassign a null value to iNaT - if isna(value): + if isna(value) and not isinstance(value, np.datetime64): + # exclude np.datetime64("NaT") value = iNaT try: diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index c8342c54e9b5d..1fa0f137ef66b 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -654,6 +654,41 @@ def test_timedelta_assignment(): tm.assert_series_equal(s, expected) +def test_timedelta_nat_assignment_series(): + base = pd.Series([0, 1, 2], dtype='m8[ns]') + expected = pd.Series([pd.NaT, 1, 2], dtype='m8[ns]') + + casting_nas = [pd.NaT, np.timedelta64('NaT', 'ns')] + for nat in casting_nas: + ser = base.copy(deep=True) + ser[0] = nat + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat + tm.assert_series_equal(ser, expected) + + # a specifically-datetime NaT should not be coerced to timedelta + expected = pd.Series([ + pd.NaT, pd.Timedelta(nanoseconds=1), pd.Timedelta(nanoseconds=2)], + dtype=object + ) + + non_casting_nas = [np.datetime64('NaT', 'ns')] + for nat in non_casting_nas: + ser = base.copy(deep=True) + ser[0] = nat + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.iloc[0] = nat + tm.assert_series_equal(ser, expected) + + def test_underlying_data_conversion(): # GH 4080 df = DataFrame({c: [1, 2, 3] for c in ["a", "b", "c"]}) From 511650c0bb1357c0d80f9105c891aed4ac53074b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 09:49:54 -0700 Subject: [PATCH 02/11] BUG: fix+test setting datetime64(NaT) in TimedeltaArray --- pandas/core/arrays/datetimelike.py | 26 ++++++++++++++- pandas/tests/series/indexing/test_indexing.py | 32 +++++++++++++++---- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 540442b7eaed4..a1d2f38268e1b 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -492,7 +492,7 @@ def __setitem__( elif isinstance(value, self._scalar_type): self._check_compatible_with(value) value = self._unbox_scalar(value) - elif isna(value) or value == iNaT: + elif is_valid_na(value, self.dtype) or value == iNaT: value = iNaT else: msg = ( @@ -1518,6 +1518,30 @@ def mean(self, skipna=True): return self._box_func(result) +def is_valid_na(obj, dtype): + """ + isna check that excludes incompatible dtypes + + Parameters + ---------- + obj : object + dtype : np.datetime64, np.timedelta64, DatetimeTZDtype, or PeriodDtype + + Returns + ------- + bool + """ + if not isna(obj): + return False + if dtype.kind == "M": + return not isinstance(obj, np.timedelta64) + if dtype.kind == "m": + return not isinstance(obj, np.datetime64) + + # must be PeriodDType + return not isinstance(obj, (np.datetime64, np.timedelta64)) + + # ------------------------------------------------------------------- # Shared Constructor Helpers diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 1fa0f137ef66b..0a44c04fb6c1d 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -655,10 +655,10 @@ def test_timedelta_assignment(): def test_timedelta_nat_assignment_series(): - base = pd.Series([0, 1, 2], dtype='m8[ns]') - expected = pd.Series([pd.NaT, 1, 2], dtype='m8[ns]') + base = pd.Series([0, 1, 2], dtype="m8[ns]") + expected = pd.Series([pd.NaT, 1, 2], dtype="m8[ns]") - casting_nas = [pd.NaT, np.timedelta64('NaT', 'ns')] + casting_nas = [pd.NaT, np.timedelta64("NaT", "ns")] for nat in casting_nas: ser = base.copy(deep=True) ser[0] = nat @@ -669,12 +669,11 @@ def test_timedelta_nat_assignment_series(): tm.assert_series_equal(ser, expected) # a specifically-datetime NaT should not be coerced to timedelta - expected = pd.Series([ - pd.NaT, pd.Timedelta(nanoseconds=1), pd.Timedelta(nanoseconds=2)], - dtype=object + expected = pd.Series( + [pd.NaT, pd.Timedelta(nanoseconds=1), pd.Timedelta(nanoseconds=2)], dtype=object ) - non_casting_nas = [np.datetime64('NaT', 'ns')] + non_casting_nas = [np.datetime64("NaT", "ns")] for nat in non_casting_nas: ser = base.copy(deep=True) ser[0] = nat @@ -689,6 +688,25 @@ def test_timedelta_nat_assignment_series(): tm.assert_series_equal(ser, expected) +def test_timedelta_nat_assignment_array(): + tdi = pd.timedelta_range("1s", periods=3, freq="s") + tda = tdi._data + expected = pd.TimedeltaIndex([pd.NaT, tdi[1], tdi[2]])._data + + casting_nas = [pd.NaT, np.timedelta64("NaT", "ns")] + for nat in casting_nas: + arr = tda.copy() + arr[0] = nat + + tm.assert_equal(arr, expected) + + non_casting_nas = [np.datetime64("NaT", "ns")] + for nat in non_casting_nas: + arr = tda.copy() + with pytest.raises(TypeError): + arr[0] = nat + + def test_underlying_data_conversion(): # GH 4080 df = DataFrame({c: [1, 2, 3] for c in ["a", "b", "c"]}) From 3c916c1c246555e56339d78a6d323ff882d3f810 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 09:59:15 -0700 Subject: [PATCH 03/11] TST: inserting timedelta64(NaT) into DatetimeArray, fixed in previous commit --- pandas/tests/series/indexing/test_indexing.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 0a44c04fb6c1d..7609d3156bed5 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -655,8 +655,9 @@ def test_timedelta_assignment(): def test_timedelta_nat_assignment_series(): - base = pd.Series([0, 1, 2], dtype="m8[ns]") - expected = pd.Series([pd.NaT, 1, 2], dtype="m8[ns]") + tdi = pd.timedelta_range("1s", periods=3, freq="s") + base = pd.Series(tdi) + expected = pd.Series([pd.NaT, tdi[1], tdi[2]], dtype="m8[ns]") casting_nas = [pd.NaT, np.timedelta64("NaT", "ns")] for nat in casting_nas: @@ -669,10 +670,7 @@ def test_timedelta_nat_assignment_series(): tm.assert_series_equal(ser, expected) # a specifically-datetime NaT should not be coerced to timedelta - expected = pd.Series( - [pd.NaT, pd.Timedelta(nanoseconds=1), pd.Timedelta(nanoseconds=2)], dtype=object - ) - + expected = expected.astype(object) non_casting_nas = [np.datetime64("NaT", "ns")] for nat in non_casting_nas: ser = base.copy(deep=True) @@ -707,6 +705,26 @@ def test_timedelta_nat_assignment_array(): arr[0] = nat +@pytest.mark.parametrize("tz", [None, "US/Pacific"]) +def test_datetime_nat_assignment_array(tz): + dti = pd.date_range("2016-01-01", periods=3, tz=tz) + dta = dti._data + expected = pd.DatetimeIndex([pd.NaT, dti[1], dti[2]])._data + + casting_nas = [pd.NaT, np.datetime64("NaT", "ns")] + for nat in casting_nas: + arr = dta.copy() + arr[0] = nat + + tm.assert_equal(arr, expected) + + non_casting_nas = [np.timedelta64("NaT", "ns")] + for nat in non_casting_nas: + arr = dta.copy() + with pytest.raises(TypeError): + arr[0] = nat + + def test_underlying_data_conversion(): # GH 4080 df = DataFrame({c: [1, 2, 3] for c in ["a", "b", "c"]}) From d39503db9c610bc487336ded2d4ad67becbbad82 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 10:04:37 -0700 Subject: [PATCH 04/11] BUG: fix+test assigning timedelta64(NaT) to datetime series --- pandas/_libs/index.pyx | 4 ++- pandas/core/internals/blocks.py | 6 ++-- pandas/tests/series/indexing/test_indexing.py | 36 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx index fc543c0bfa322..22d9bbb4e1307 100644 --- a/pandas/_libs/index.pyx +++ b/pandas/_libs/index.pyx @@ -534,7 +534,9 @@ cpdef convert_scalar(ndarray arr, object value): elif isinstance(value, (datetime, np.datetime64, date)): return Timestamp(value).value elif value is None or value != value: - return NPY_NAT + if not util.is_timedelta64_object(value): + # exclude np.timedelta64("NaT") + return NPY_NAT elif isinstance(value, str): return Timestamp(value).value raise ValueError("cannot set a Timestamp with a non-timestamp") diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 7e42df2cfb0f3..03f90d4f9dbef 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2295,7 +2295,8 @@ def _try_coerce_args(self, other): ------- base-type other """ - if is_null_datetimelike(other): + if is_null_datetimelike(other) and not isinstance(other, np.timedelta64): + # exclude np.timedelta64("NaT") other = tslibs.iNaT elif isinstance(other, (datetime, np.datetime64, date)): other = self._box_func(other) @@ -2485,7 +2486,8 @@ def _try_coerce_args(self, other): # add the tz back other = self._holder(other, dtype=self.dtype) - elif is_null_datetimelike(other): + if is_null_datetimelike(other) and not isinstance(other, np.timedelta64): + # exclude np.timedelta64("NaT") other = tslibs.iNaT elif isinstance(other, self._holder): if other.tz != self.values.tz: diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 7609d3156bed5..448eb668629a8 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -686,6 +686,42 @@ def test_timedelta_nat_assignment_series(): tm.assert_series_equal(ser, expected) +@pytest.mark.parametrize("tz", [None, "US/Pacific"]) +def test_datetime_nat_assignment_series(tz): + dti = pd.date_range("2016-01-01", periods=3, tz=tz) + base = pd.Series(dti) + expected = pd.Series([pd.NaT, dti[1], dti[2]], dtype=dti.dtype) + + casting_nas = [pd.NaT, np.datetime64("NaT", "ns")] + for nat in casting_nas: + ser = base.copy(deep=True) + ser[0] = nat + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat + tm.assert_series_equal(ser, expected) + + # a specifically-datetime NaT should not be coerced to timedelta + expected = expected.astype(object) + non_casting_nas = [np.timedelta64("NaT", "ns")] + for nat in non_casting_nas: + expected[0] = nat + assert expected[0] is nat + + ser = base.copy(deep=True) + ser[0] = nat + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.iloc[0] = nat + tm.assert_series_equal(ser, expected) + + def test_timedelta_nat_assignment_array(): tdi = pd.timedelta_range("1s", periods=3, freq="s") tda = tdi._data From b047cceb4018bcfd3b7ccb7f286110fd865052fe Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 11:26:33 -0700 Subject: [PATCH 05/11] BUG: fix+test assignment of wrong nat to DataFrame --- pandas/core/internals/blocks.py | 5 ++- pandas/tests/series/indexing/test_indexing.py | 34 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 03f90d4f9dbef..6c90a5f11a9bc 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2591,8 +2591,11 @@ def setitem(self, indexer, value): try: return super().setitem(indexer, value) except (ValueError, TypeError): + obj_vals = self.values.astype(object) + if self.ndim == 2 and obj_vals.ndim == 1: + obj_vals = obj_vals.reshape(1, -1) newb = make_block( - self.values.astype(object), placement=self.mgr_locs, klass=ObjectBlock + obj_vals, placement=self.mgr_locs, klass=ObjectBlock, ndim=self.ndim ) return newb.setitem(indexer, value) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 448eb668629a8..60188f3dfc8c7 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -669,6 +669,14 @@ def test_timedelta_nat_assignment_series(): ser.loc[0] = nat tm.assert_series_equal(ser, expected) + df = base.copy(deep=True).to_frame() + df.loc[0, 0] = nat + tm.assert_frame_equal(df, expected.to_frame()) + + df = base.copy(deep=True).to_frame() + df.iloc[0, 0] = nat + tm.assert_frame_equal(df, expected.to_frame()) + # a specifically-datetime NaT should not be coerced to timedelta expected = expected.astype(object) non_casting_nas = [np.datetime64("NaT", "ns")] @@ -685,6 +693,14 @@ def test_timedelta_nat_assignment_series(): ser.iloc[0] = nat tm.assert_series_equal(ser, expected) + df = base.copy(deep=True).to_frame() + df.loc[0, 0] = nat + tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) + + df = base.copy(deep=True).to_frame() + df.iloc[0, 0] = nat + tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) + @pytest.mark.parametrize("tz", [None, "US/Pacific"]) def test_datetime_nat_assignment_series(tz): @@ -702,7 +718,15 @@ def test_datetime_nat_assignment_series(tz): ser.loc[0] = nat tm.assert_series_equal(ser, expected) - # a specifically-datetime NaT should not be coerced to timedelta + df = base.copy(deep=True).to_frame() + df.loc[0, 0] = nat + tm.assert_frame_equal(df, expected.to_frame()) + + df = base.copy(deep=True).to_frame() + df.iloc[0, 0] = nat + tm.assert_frame_equal(df, expected.to_frame()) + + # a specifically-timedelta NaT should not be coerced to datetime expected = expected.astype(object) non_casting_nas = [np.timedelta64("NaT", "ns")] for nat in non_casting_nas: @@ -721,6 +745,14 @@ def test_datetime_nat_assignment_series(tz): ser.iloc[0] = nat tm.assert_series_equal(ser, expected) + df = base.copy(deep=True).to_frame() + df.loc[0, 0] = nat + tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) + + df = base.copy(deep=True).to_frame() + df.iloc[0, 0] = nat + tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) + def test_timedelta_nat_assignment_array(): tdi = pd.timedelta_range("1s", periods=3, freq="s") From 217ce6304c9eeba17cf713586bd8aabe76d24818 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 15 Jul 2019 13:55:13 -0700 Subject: [PATCH 06/11] pre-rebase --- pandas/_libs/index.pyx | 14 ++++++++------ pandas/core/dtypes/missing.py | 2 +- pandas/core/internals/blocks.py | 20 +++++++++++++++----- pandas/core/series.py | 6 ++++-- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx index 22d9bbb4e1307..4237c498348ee 100644 --- a/pandas/_libs/index.pyx +++ b/pandas/_libs/index.pyx @@ -533,10 +533,11 @@ cpdef convert_scalar(ndarray arr, object value): pass elif isinstance(value, (datetime, np.datetime64, date)): return Timestamp(value).value + elif util.is_timedelta64_object(value): + # exclude np.timedelta64("NaT") from value != value below + pass elif value is None or value != value: - if not util.is_timedelta64_object(value): - # exclude np.timedelta64("NaT") - return NPY_NAT + return NPY_NAT elif isinstance(value, str): return Timestamp(value).value raise ValueError("cannot set a Timestamp with a non-timestamp") @@ -546,10 +547,11 @@ cpdef convert_scalar(ndarray arr, object value): pass elif isinstance(value, timedelta): return Timedelta(value).value + elif util.is_datetime64_object(value): + # exclude np.datetime64("NaT") from value != value below + pass elif value is None or value != value: - if not util.is_datetime64_object(value): - # exclude np.datetime64("NaT") - return NPY_NAT + return NPY_NAT elif isinstance(value, str): return Timedelta(value).value raise ValueError("cannot set a Timedelta with a non-timedelta") diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index 6a681954fd902..e4e435f09f7a8 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -574,7 +574,7 @@ def is_valid_nat_for_dtype(obj, dtype): ------- bool """ - if not isna(obj): + if not is_scalar(obj) or not isna(obj): return False if dtype.kind == "M": return not isinstance(obj, np.timedelta64) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 9e2712fa68742..446f425bb2a8c 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -9,7 +9,7 @@ from pandas._libs import NaT, lib, tslib, tslibs import pandas._libs.internals as libinternals -from pandas._libs.tslibs import Timedelta, conversion, is_null_datetimelike +from pandas._libs.tslibs import Timedelta, conversion from pandas.util._validators import validate_bool_kwarg from pandas.core.dtypes.cast import ( @@ -60,7 +60,13 @@ ABCPandasArray, ABCSeries, ) -from pandas.core.dtypes.missing import _isna_compat, array_equivalent, isna, notna +from pandas.core.dtypes.missing import ( + _isna_compat, + array_equivalent, + is_valid_nat_for_dtype, + isna, + notna, +) import pandas.core.algorithms as algos from pandas.core.arrays import ( @@ -2275,7 +2281,7 @@ def _try_coerce_args(self, other): ------- base-type other """ - if is_null_datetimelike(other) and not isinstance(other, np.timedelta64): + if is_valid_nat_for_dtype(other, self.dtype): # exclude np.timedelta64("NaT") other = tslibs.iNaT elif isinstance(other, (datetime, np.datetime64, date)): @@ -2285,6 +2291,8 @@ def _try_coerce_args(self, other): other = other.asm8.view("i8") elif hasattr(other, "dtype") and is_datetime64_dtype(other): other = other.astype("i8", copy=False).view("i8") + elif is_integer(other) and other == tslibs.iNaT: + pass else: # coercion issues # let higher levels handle @@ -2466,7 +2474,7 @@ def _try_coerce_args(self, other): # add the tz back other = self._holder(other, dtype=self.dtype) - if is_null_datetimelike(other) and not isinstance(other, np.timedelta64): + if is_valid_nat_for_dtype(other, self.dtype): # exclude np.timedelta64("NaT") other = tslibs.iNaT elif isinstance(other, self._holder): @@ -2650,13 +2658,15 @@ def _try_coerce_args(self, other): base-type other """ - if is_null_datetimelike(other) and not isinstance(other, np.datetime64): + if is_valid_nat_for_dtype(other, self.dtype): # exclude np.datetime64("NaT") other = tslibs.iNaT elif isinstance(other, (timedelta, np.timedelta64)): other = Timedelta(other).value elif hasattr(other, "dtype") and is_timedelta64_dtype(other): other = other.astype("i8", copy=False).view("i8") + elif is_integer(other) and other == tslibs.iNaT: + pass else: # coercion issues # let higher levels handle diff --git a/pandas/core/series.py b/pandas/core/series.py index 121c1a8b8256c..33734fb50ffc7 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -47,6 +47,7 @@ ) from pandas.core.dtypes.missing import ( isna, + is_valid_nat_for_dtype, na_value_for_dtype, notna, remove_na_arraylike, @@ -1198,14 +1199,15 @@ def setitem(key, value): pass elif is_timedelta64_dtype(self.dtype): # reassign a null value to iNaT - if isna(value) and not isinstance(value, np.datetime64): + if is_valid_nat_for_dtype(value, self.dtype): # exclude np.datetime64("NaT") value = iNaT try: self.index._engine.set_value(self._values, key, value) return - except TypeError: + except (TypeError, ValueError): + # ValueError appears in only some build in CI pass self.loc[key] = value From d32c1bfef9422e21080c48713bcf79d9c6f55c73 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 15 Jul 2019 14:35:53 -0700 Subject: [PATCH 07/11] typo --- pandas/core/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index 33734fb50ffc7..1d4b7386ae2e7 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1207,7 +1207,7 @@ def setitem(key, value): self.index._engine.set_value(self._values, key, value) return except (TypeError, ValueError): - # ValueError appears in only some build in CI + # ValueError appears in only some builds in CI pass self.loc[key] = value From c001aca13f5057852fe28b2af5bd7cc9ea4f270b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 17 Jul 2019 15:26:18 -0700 Subject: [PATCH 08/11] update test for datetime series --- pandas/tests/series/indexing/test_indexing.py | 124 ++++-------------- 1 file changed, 28 insertions(+), 96 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 6c097d15773b6..2d36bfdb93a17 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -654,104 +654,36 @@ def test_timedelta_assignment(): tm.assert_series_equal(s, expected) -def test_timedelta_nat_assignment_series(): - tdi = pd.timedelta_range("1s", periods=3, freq="s") - base = pd.Series(tdi) - expected = pd.Series([pd.NaT, tdi[1], tdi[2]], dtype="m8[ns]") - - casting_nas = [pd.NaT, np.timedelta64("NaT", "ns")] - for nat in casting_nas: - ser = base.copy(deep=True) - ser[0] = nat - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.loc[0] = nat - tm.assert_series_equal(ser, expected) - - df = base.copy(deep=True).to_frame() - df.loc[0, 0] = nat - tm.assert_frame_equal(df, expected.to_frame()) - - df = base.copy(deep=True).to_frame() - df.iloc[0, 0] = nat - tm.assert_frame_equal(df, expected.to_frame()) - - # a specifically-datetime NaT should not be coerced to timedelta - expected = expected.astype(object) - non_casting_nas = [np.datetime64("NaT", "ns")] - for nat in non_casting_nas: - ser = base.copy(deep=True) - ser[0] = nat - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.loc[0] = nat - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.iloc[0] = nat - tm.assert_series_equal(ser, expected) - - df = base.copy(deep=True).to_frame() - df.loc[0, 0] = nat - tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) - - df = base.copy(deep=True).to_frame() - df.iloc[0, 0] = nat - tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) - - -@pytest.mark.parametrize("tz", [None, "US/Pacific"]) -def test_datetime_nat_assignment_series(tz): +@pytest.mark.parametrize( + "nat_val,should_cast", + [ + (pd.NaT, True), + (np.timedelta64("NaT", "ns"), False), + (np.datetime64("NaT", "ns"), True), + ], +) +@pytest.mark.parametrize("tz", [None, "UTC"]) +def test_dt64_series_assign_nat(nat_val, should_cast, tz): + # some nat-like values should be cast to datetime64 when inserting + # into a datetime64 series. Others should coerce to object + # and retain their dtypes. dti = pd.date_range("2016-01-01", periods=3, tz=tz) base = pd.Series(dti) - expected = pd.Series([pd.NaT, dti[1], dti[2]], dtype=dti.dtype) - - casting_nas = [pd.NaT, np.datetime64("NaT", "ns")] - for nat in casting_nas: - ser = base.copy(deep=True) - ser[0] = nat - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.loc[0] = nat - tm.assert_series_equal(ser, expected) - - df = base.copy(deep=True).to_frame() - df.loc[0, 0] = nat - tm.assert_frame_equal(df, expected.to_frame()) - - df = base.copy(deep=True).to_frame() - df.iloc[0, 0] = nat - tm.assert_frame_equal(df, expected.to_frame()) - - # a specifically-timedelta NaT should not be coerced to datetime - expected = expected.astype(object) - non_casting_nas = [np.timedelta64("NaT", "ns")] - for nat in non_casting_nas: - expected[0] = nat - assert expected[0] is nat - - ser = base.copy(deep=True) - ser[0] = nat - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.loc[0] = nat - tm.assert_series_equal(ser, expected) - - ser = base.copy(deep=True) - ser.iloc[0] = nat - tm.assert_series_equal(ser, expected) - - df = base.copy(deep=True).to_frame() - df.loc[0, 0] = nat - tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) - - df = base.copy(deep=True).to_frame() - df.iloc[0, 0] = nat - tm.assert_frame_equal(df, pd.DataFrame(expected, dtype=object)) + expected = pd.Series([pd.NaT] + list(dti[1:]), dtype=dti.dtype) + if not should_cast: + expected = expected.astype(object) + + ser = base.copy(deep=True) + ser[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.iloc[0] = nat_val + tm.assert_series_equal(ser, expected) @pytest.mark.parametrize( From 49eec2e693f394084eedc9b72fd835f9bdaebd38 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 17 Jul 2019 15:28:08 -0700 Subject: [PATCH 09/11] fixups --- pandas/core/internals/blocks.py | 1 + pandas/core/series.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 663983d3c73dc..36b823d686ba7 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2478,6 +2478,7 @@ def _try_coerce_args(self, other): elif is_datetime64_dtype(other): # add the tz back other = self._holder(other, dtype=self.dtype) + elif is_valid_nat_for_dtype(other, self.dtype): other = tslibs.iNaT elif is_integer(other) and other == tslibs.iNaT: diff --git a/pandas/core/series.py b/pandas/core/series.py index ccef46aff60c0..0f0914a4f74aa 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -48,7 +48,6 @@ from pandas.core.dtypes.missing import ( is_valid_nat_for_dtype, isna, - is_valid_nat_for_dtype, na_value_for_dtype, notna, remove_na_arraylike, From 7c260d50bf6e2aa065fbeb4b022e81f2aeaada8b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 20 Jul 2019 17:09:22 -0700 Subject: [PATCH 10/11] remove apparently unnecessary reshape --- pandas/core/internals/blocks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 36b823d686ba7..5dc7f2f7eebcf 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2586,8 +2586,6 @@ def setitem(self, indexer, value): return super().setitem(indexer, value) except (ValueError, TypeError): obj_vals = self.values.astype(object) - if self.ndim == 2 and obj_vals.ndim == 1: - obj_vals = obj_vals.reshape(1, -1) newb = make_block( obj_vals, placement=self.mgr_locs, klass=ObjectBlock, ndim=self.ndim ) From fbc18368addb179d1c79301ba7fff3e852efe616 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 22 Jul 2019 07:30:53 -0700 Subject: [PATCH 11/11] whatsnew --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 9caf127553e05..87d3e47cbefe5 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -87,7 +87,7 @@ Categorical Datetimelike ^^^^^^^^^^^^ - +- Bug in :meth:`Series.__setitem__` incorrectly casting ``np.timedelta64("NaT")`` to ``np.datetime64("NaT")`` when inserting into a :class:`Series` with datetime64 dtype (:issue:`27311`) - -