From 7b3241221cbec76ed92b9692500ac80093edf012 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 24 Feb 2019 17:31:22 -0800 Subject: [PATCH 1/6] BUG: Fix formatting of np.NaT --- pandas/io/formats/format.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index f68ef2cc39006..5d4d8b4a470e2 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -942,11 +942,20 @@ def _format_strings(self): self.formatter if self.formatter is not None else (lambda x: pprint_thing(x, escape_chars=('\t', '\r', '\n')))) + def is_nat(x): + # Compat for numpy 1.12. Replace with np.isnat once min dep is 1.13 + try: + return np.isnat(x) + except AttributeError: + return str(x) == 'NaT' + except TypeError: + return False + def _format(x): if self.na_rep is not None and is_scalar(x) and isna(x): if x is None: return 'None' - elif x is NaT: + elif x is NaT or is_nat(x): return 'NaT' return self.na_rep elif isinstance(x, PandasObject): From 83ec1de10e983235babcfadfc8be4f2f54f283de Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 25 Feb 2019 23:14:16 -0800 Subject: [PATCH 2/6] BUG: repr of np.datetime64('NaT') in Series/DataFrame with dtype object --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/_libs/tslibs/nattype.pyx | 12 ++++++++++++ pandas/io/formats/format.py | 12 ++---------- pandas/tests/frame/test_repr_info.py | 8 ++++++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 170e7f14da397..b9209bdb10d24 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -233,7 +233,7 @@ Sparse Other ^^^^^ -- +- Bug in :class:`Series` and :class:`DataFrame` repr where ``np.datetime64('NaT')`` and ``np.timedelta64('NaT')`` with ``dtype=object`` would be represented as ``NaN`` (:issue:``) - - diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 79e2e256c501d..31c5b828e57ac 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -39,6 +39,18 @@ _nat_scalar_rules[Py_GE] = False # ---------------------------------------------------------------------- +cpdef bint is_np_nat(x): + """Compat check for np.datetime('NaT')""" + try: + return np.isnat(x) + except AttributeError: + # numpy 1.12 compat + return str(x) == 'NaT' + except TypeError: + # np.isnat only defined for datetime, timedelta + return False + + def _make_nan_func(func_name, doc): def f(*args, **kwargs): return np.nan diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 5d4d8b4a470e2..d93b3d2d0b5b2 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -13,6 +13,7 @@ from pandas._libs import lib from pandas._libs.tslib import format_array_from_datetime from pandas._libs.tslibs import NaT, Timedelta, Timestamp, iNaT +from pandas._libs.tslibs.nattype import is_np_nat from pandas.compat import StringIO, lzip, map, u, zip from pandas.core.dtypes.common import ( @@ -942,20 +943,11 @@ def _format_strings(self): self.formatter if self.formatter is not None else (lambda x: pprint_thing(x, escape_chars=('\t', '\r', '\n')))) - def is_nat(x): - # Compat for numpy 1.12. Replace with np.isnat once min dep is 1.13 - try: - return np.isnat(x) - except AttributeError: - return str(x) == 'NaT' - except TypeError: - return False - def _format(x): if self.na_rep is not None and is_scalar(x) and isna(x): if x is None: return 'None' - elif x is NaT or is_nat(x): + elif x is NaT or is_np_nat(x): return 'NaT' return self.na_rep elif isinstance(x, PandasObject): diff --git a/pandas/tests/frame/test_repr_info.py b/pandas/tests/frame/test_repr_info.py index 4a7cb7f508926..ccc5d52340491 100644 --- a/pandas/tests/frame/test_repr_info.py +++ b/pandas/tests/frame/test_repr_info.py @@ -521,3 +521,11 @@ def test_repr_categorical_dates_periods(self): df = DataFrame({'dt': Categorical(dt), 'p': Categorical(p)}) assert repr(df) == exp + + @pytest.mark.parametrize('arg', [np.datetime64, np.timedelta64]) + @pytest.mark.parametrize('box, expected', [ + [Series, '0 NaT\ndtype: object'], + [DataFrame, ' 0\n0 NaT']]) + def test_repr_np_nat_with_object(self, arg, box, expected): + result = repr(box([arg('NaT')], dtype=object)) + assert result == expected From e67403ed729ae3be8c96691bfd060b2786aebcd9 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 25 Feb 2019 23:18:24 -0800 Subject: [PATCH 3/6] Add pr number --- doc/source/whatsnew/v0.25.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index b9209bdb10d24..f899c1494e69f 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -233,7 +233,7 @@ Sparse Other ^^^^^ -- Bug in :class:`Series` and :class:`DataFrame` repr where ``np.datetime64('NaT')`` and ``np.timedelta64('NaT')`` with ``dtype=object`` would be represented as ``NaN`` (:issue:``) +- Bug in :class:`Series` and :class:`DataFrame` repr where ``np.datetime64('NaT')`` and ``np.timedelta64('NaT')`` with ``dtype=object`` would be represented as ``NaN`` (:issue:`25445`) - - From b7aaa0262a0462010cf385a2921a0f4d92c38158 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 26 Feb 2019 08:28:07 -0800 Subject: [PATCH 4/6] Handle compat error --- pandas/_libs/tslibs/nattype.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 31c5b828e57ac..eceb8576629a9 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -46,7 +46,7 @@ cpdef bint is_np_nat(x): except AttributeError: # numpy 1.12 compat return str(x) == 'NaT' - except TypeError: + except (TypeError, ValueError): # np.isnat only defined for datetime, timedelta return False From 2ab9f1424f4f1877afc68533d4a8ceb5e0200a25 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 28 Mar 2019 15:16:16 -0700 Subject: [PATCH 5/6] Remove need for a function --- pandas/_libs/tslibs/nattype.pyx | 12 ------------ pandas/io/formats/format.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index c41f306059246..e71f7dfe8b8ce 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -39,18 +39,6 @@ _nat_scalar_rules[Py_GE] = False # ---------------------------------------------------------------------- -cpdef bint is_np_nat(x): - """Compat check for np.datetime('NaT')""" - try: - return np.isnat(x) - except AttributeError: - # numpy 1.12 compat - return str(x) == 'NaT' - except (TypeError, ValueError): - # np.isnat only defined for datetime, timedelta - return False - - def _make_nan_func(func_name, doc): def f(*args, **kwargs): return np.nan diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index dc8d24457aa56..0ba8d28f3f0cb 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -942,10 +942,14 @@ def _format_strings(self): def _format(x): if self.na_rep is not None and is_scalar(x) and isna(x): - if x is None: - return 'None' - elif x is NaT or is_np_nat(x): - return 'NaT' + try: + if x is None: + return 'None' + elif x is NaT or np.isnat(x): + return 'NaT' + except (TypeError, ValueError): + # np.isnat only handles datetime or timedelta objects + pass return self.na_rep elif isinstance(x, PandasObject): return '{x}'.format(x=x) From 8d8abd16a4f189b443045878b46121da935e5b8e Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 29 Mar 2019 10:13:48 -0700 Subject: [PATCH 6/6] Address comments --- pandas/io/formats/format.py | 3 ++- pandas/tests/frame/test_repr_info.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 0ba8d28f3f0cb..1d08d559cb33f 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -14,7 +14,6 @@ from pandas._libs import lib from pandas._libs.tslib import format_array_from_datetime from pandas._libs.tslibs import NaT, Timedelta, Timestamp, iNaT -from pandas._libs.tslibs.nattype import is_np_nat from pandas.compat import StringIO, lzip from pandas.core.dtypes.common import ( @@ -943,6 +942,8 @@ def _format_strings(self): def _format(x): if self.na_rep is not None and is_scalar(x) and isna(x): try: + # try block for np.isnat specifically + # determine na_rep if x is None or NaT-like if x is None: return 'None' elif x is NaT or np.isnat(x): diff --git a/pandas/tests/frame/test_repr_info.py b/pandas/tests/frame/test_repr_info.py index 0796f41bc8793..e4f0b4c6459ae 100644 --- a/pandas/tests/frame/test_repr_info.py +++ b/pandas/tests/frame/test_repr_info.py @@ -518,5 +518,6 @@ def test_repr_categorical_dates_periods(self): [Series, '0 NaT\ndtype: object'], [DataFrame, ' 0\n0 NaT']]) def test_repr_np_nat_with_object(self, arg, box, expected): + # GH 25445 result = repr(box([arg('NaT')], dtype=object)) assert result == expected