Merging from master

Author: datapythonista

doc/source/user_guide/text.rst

@@ -87,8 +87,9 @@ l. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
 
    .. ipython:: python
 
-      s.astype(object).str.count("a")
-      s.astype(object).dropna().str.count("a")
+      s2 = pd.Series(["a", None, "b"], dtype="object")
+      s2.str.count("a")
+      s2.dropna().str.count("a")
 
    When NA values are present, the output dtype is float64. Similarly for
    methods returning boolean values.

doc/source/whatsnew/v1.0.0.rst

@@ -227,6 +227,7 @@ Other enhancements
 - Added new writer for exporting Stata dta files in version 118, ``StataWriter118``.  This format supports exporting strings containing Unicode characters (:issue:`23573`)
 - :meth:`Series.map` now accepts ``collections.abc.Mapping`` subclasses as a mapper (:issue:`29733`)
 - The ``pandas.datetime`` class is now deprecated. Import from ``datetime`` instead (:issue:`30296`)
+- :meth:`Timestamp.fromisocalendar` is now compatible with python 3.8 and above (:issue:`28115`)
 
 
 
@@ -836,6 +837,7 @@ Datetimelike
 - Bug in :class:`DatetimeArray`, :class:`TimedeltaArray`, and :class:`PeriodArray` where inplace addition and subtraction did not actually operate inplace (:issue:`24115`)
 - Bug in :func:`pandas.to_datetime` when called with ``Series`` storing ``IntegerArray`` raising ``TypeError`` instead of returning ``Series`` (:issue:`30050`)
 - Bug in :func:`date_range` with custom business hours as ``freq`` and given number of ``periods`` (:issue:`30593`)
+- Bug in :class:`PeriodIndex` comparisons with incorrectly casting integers to :class:`Period` objects, inconsistent with the :class:`Period` comparison behavior (:issue:`30722`)
 
 Timedelta
 ^^^^^^^^^
@@ -885,6 +887,7 @@ Interval
 
 - Bug in :meth:`IntervalIndex.get_indexer` where a :class:`Categorical` or :class:`CategoricalIndex` ``target`` would incorrectly raise a ``TypeError`` (:issue:`30063`)
 - Bug in ``pandas.core.dtypes.cast.infer_dtype_from_scalar`` where passing ``pandas_dtype=True`` did not infer :class:`IntervalDtype` (:issue:`30337`)
+- Bug in :class:`Series` constructor where constructing a ``Series`` from a ``list`` of :class:`Interval` objects resulted in ``object`` dtype instead of :class:`IntervalDtype` (:issue:`23563`)
 - Bug in :class:`IntervalDtype` where the ``kind`` attribute was incorrectly set as ``None`` instead of ``"O"`` (:issue:`30568`)
 - Bug in :class:`IntervalIndex`, :class:`~arrays.IntervalArray`, and :class:`Series` with interval data where equality comparisons were incorrect (:issue:`24112`)
 
@@ -1001,7 +1004,7 @@ Reshaping
 Sparse
 ^^^^^^
 - Bug in :class:`SparseDataFrame` arithmetic operations incorrectly casting inputs to float (:issue:`28107`)
--
+- Bug in ``DataFrame.sparse`` returning a ``Series`` when there was a column named ``sparse`` rather than the accessor (:issue:`30758`)
 -
 
 ExtensionArray

pandas/_libs/interval.pyx

@@ -326,7 +326,7 @@ cdef class Interval(IntervalMixin):
     def __hash__(self):
         return hash((self.left, self.right, self.closed))
 
-    def __contains__(self, key):
+    def __contains__(self, key) -> bool:
         if _interval_like(key):
             raise TypeError("__contains__ not defined for two intervals")
         return ((self.left < key if self.open_left else self.left <= key) and

pandas/_libs/tslib.pyx

@@ -120,8 +120,7 @@ def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
     elif box == "datetime":
         func_create = create_datetime_from_ts
     else:
-        raise ValueError("box must be one of 'datetime', 'date', 'time' or"
-                         " 'timestamp'")
+        raise ValueError("box must be one of 'datetime', 'date', 'time' or 'timestamp'")
 
     if is_utc(tz) or tz is None:
         for i in range(n):

pandas/_libs/tslibs/nattype.pyx

@@ -5,6 +5,9 @@ from cpython.object cimport (
 from cpython.datetime cimport (datetime,
                                PyDateTime_Check, PyDelta_Check,
                                PyDateTime_IMPORT)
+
+from cpython.version cimport PY_MINOR_VERSION
+
 PyDateTime_IMPORT
 
 import numpy as np
@@ -19,6 +22,7 @@ from pandas._libs.tslibs.util cimport (
     get_nat, is_integer_object, is_float_object, is_datetime64_object,
     is_timedelta64_object)
 
+
 # ----------------------------------------------------------------------
 # Constants
 nat_strings = {'NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN'}
@@ -427,6 +431,10 @@ class NaTType(_NaT):
     tzname = _make_error_func('tzname', datetime)
     utcoffset = _make_error_func('utcoffset', datetime)
 
+    # "fromisocalendar" was introduced in 3.8
+    if PY_MINOR_VERSION >= 8:
+        fromisocalendar = _make_error_func('fromisocalendar', datetime)
+
     # ----------------------------------------------------------------------
     # The remaining methods have docstrings copy/pasted from the analogous
     # Timestamp methods.

pandas/_libs/tslibs/strptime.pyx

@@ -278,8 +278,8 @@ def array_strptime(object[:] values, object fmt,
                                  "the ISO year directive '%G' and a weekday "
                                  "directive '%A', '%a', '%w', or '%u'.")
             else:
-                raise ValueError("ISO week directive '%V' is incompatible with"
-                                 " the year directive '%Y'. Use the ISO year "
+                raise ValueError("ISO week directive '%V' is incompatible with "
+                                 "the year directive '%Y'. Use the ISO year "
                                  "'%G' instead.")
 
         # If we know the wk of the year and what day of that wk, we can figure

pandas/_libs/tslibs/timestamps.pyx

@@ -814,9 +814,9 @@ default 'raise'
                                'shift_backward')
         if nonexistent not in nonexistent_options and not isinstance(
             nonexistent, timedelta):
-            raise ValueError("The nonexistent argument must be one of 'raise',"
-                             " 'NaT', 'shift_forward', 'shift_backward' or"
-                             " a timedelta object")
+            raise ValueError("The nonexistent argument must be one of 'raise', "
+                             "'NaT', 'shift_forward', 'shift_backward' or "
+                             "a timedelta object")
 
         if self.tzinfo is None:
             # tz naive, localize

pandas/_testing.py

@@ -1287,7 +1287,7 @@ def assert_frame_equal(
     >>> assert_frame_equal(df1, df2)
     Traceback (most recent call last):
     ...
-    AssertionError: Attributes of DataFrame.iloc[:, 1] are different
+    AssertionError: Attributes of DataFrame.iloc[:, 1] (column name="b") are different
 
     Attribute "dtype" are different
     [left]:  int64
@@ -1366,7 +1366,7 @@ def assert_frame_equal(
                 check_names=check_names,
                 check_datetimelike_compat=check_datetimelike_compat,
                 check_categorical=check_categorical,
-                obj=f"{obj}.iloc[:, {i}]",
+                obj=f'{obj}.iloc[:, {i}] (column name="{col}")',
             )
 
 

pandas/core/arrays/categorical.py

@@ -39,7 +39,7 @@
 )
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
-from pandas.core.dtypes.inference import is_hashable
+from pandas.core.dtypes.inference import is_array_like, is_hashable
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import ops
@@ -1874,7 +1874,7 @@ def __iter__(self):
         """
         return iter(self._internal_get_values().tolist())
 
-    def __contains__(self, key):
+    def __contains__(self, key) -> bool:
         """
         Returns True if `key` is in this Categorical.
         """
@@ -1884,7 +1884,7 @@ def __contains__(self, key):
 
         return contains(self, key, container=self._codes)
 
-    def _tidy_repr(self, max_vals=10, footer=True):
+    def _tidy_repr(self, max_vals=10, footer=True) -> str:
         """ a short repr displaying only max_vals and an optional (but default
         footer)
         """
@@ -1921,7 +1921,7 @@ def _repr_categories(self):
         category_strs = [x.strip() for x in category_strs]
         return category_strs
 
-    def _repr_categories_info(self):
+    def _repr_categories_info(self) -> str:
         """
         Returns a string representation of the footer.
         """
@@ -1951,11 +1951,11 @@ def _repr_categories_info(self):
         # replace to simple save space by
         return levheader + "[" + levstring.replace(" < ... < ", " ... ") + "]"
 
-    def _repr_footer(self):
+    def _repr_footer(self) -> str:
         info = self._repr_categories_info()
         return f"Length: {len(self)}\n{info}"
 
-    def _get_repr(self, length=True, na_rep="NaN", footer=True):
+    def _get_repr(self, length=True, na_rep="NaN", footer=True) -> str:
         from pandas.io.formats import format as fmt
 
         formatter = fmt.CategoricalFormatter(
@@ -1998,7 +1998,10 @@ def __getitem__(self, key):
             else:
                 return self.categories[i]
 
-        elif com.is_bool_indexer(key):
+        if is_list_like(key) and not is_array_like(key):
+            key = np.asarray(key)
+
+        if com.is_bool_indexer(key):
             key = check_bool_array_indexer(self, key)
 
         return self._constructor(

pandas/core/arrays/datetimelike.py

@@ -11,6 +11,7 @@
 from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds
 from pandas._libs.tslibs.timestamps import RoundTo, round_nsint64
 from pandas._typing import DatetimeLikeScalar
+from pandas.compat import set_function_name
 from pandas.compat.numpy import function as nv
 from pandas.errors import AbstractMethodError, NullFrequencyError, PerformanceWarning
 from pandas.util._decorators import Appender, Substitution
@@ -37,19 +38,94 @@
 from pandas.core.dtypes.inference import is_array_like
 from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna
 
-from pandas.core import missing, nanops
+from pandas.core import missing, nanops, ops
 from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts
 import pandas.core.common as com
 from pandas.core.indexers import check_bool_array_indexer
 from pandas.core.ops.common import unpack_zerodim_and_defer
-from pandas.core.ops.invalid import make_invalid_op
+from pandas.core.ops.invalid import invalid_comparison, make_invalid_op
 
 from pandas.tseries import frequencies
 from pandas.tseries.offsets import DateOffset, Tick
 
 from .base import ExtensionArray, ExtensionOpsMixin
 
 
+def _datetimelike_array_cmp(cls, op):
+    """
+    Wrap comparison operations to convert Timestamp/Timedelta/Period-like to
+    boxed scalars/arrays.
+    """
+    opname = f"__{op.__name__}__"
+    nat_result = opname == "__ne__"
+
+    @unpack_zerodim_and_defer(opname)
+    def wrapper(self, other):
+
+        if isinstance(other, str):
+            try:
+                # GH#18435 strings get a pass from tzawareness compat
+                other = self._scalar_from_string(other)
+            except ValueError:
+                # failed to parse as Timestamp/Timedelta/Period
+                return invalid_comparison(self, other, op)
+
+        if isinstance(other, self._recognized_scalars) or other is NaT:
+            other = self._scalar_type(other)
+            self._check_compatible_with(other)
+
+            other_i8 = self._unbox_scalar(other)
+
+            result = op(self.view("i8"), other_i8)
+            if isna(other):
+                result.fill(nat_result)
+
+        elif not is_list_like(other):
+            return invalid_comparison(self, other, op)
+
+        elif len(other) != len(self):
+            raise ValueError("Lengths must match")
+
+        else:
+            if isinstance(other, list):
+                # TODO: could use pd.Index to do inference?
+                other = np.array(other)
+
+            if not isinstance(other, (np.ndarray, type(self))):
+                return invalid_comparison(self, other, op)
+
+            if is_object_dtype(other):
+                # We have to use comp_method_OBJECT_ARRAY instead of numpy
+                #  comparison otherwise it would fail to raise when
+                #  comparing tz-aware and tz-naive
+                with np.errstate(all="ignore"):
+                    result = ops.comp_method_OBJECT_ARRAY(
+                        op, self.astype(object), other
+                    )
+                o_mask = isna(other)
+
+            elif not type(self)._is_recognized_dtype(other.dtype):
+                return invalid_comparison(self, other, op)
+
+            else:
+                # For PeriodDType this casting is unnecessary
+                other = type(self)._from_sequence(other)
+                self._check_compatible_with(other)
+
+                result = op(self.view("i8"), other.view("i8"))
+                o_mask = other._isnan
+
+            if o_mask.any():
+                result[o_mask] = nat_result
+
+        if self._hasnans:
+            result[self._isnan] = nat_result
+
+        return result
+
+    return set_function_name(wrapper, opname, cls)
+
+
 class AttributesMixin:
     _data: np.ndarray
 
@@ -109,7 +185,7 @@ def _unbox_scalar(self, value: Union[Period, Timestamp, Timedelta, NaTType]) ->
         raise AbstractMethodError(self)
 
     def _check_compatible_with(
-        self, other: Union[Period, Timestamp, Timedelta, NaTType]
+        self, other: Union[Period, Timestamp, Timedelta, NaTType], setitem: bool = False
     ) -> None:
         """
         Verify that `self` and `other` are compatible.
@@ -123,6 +199,9 @@ def _check_compatible_with(
         Parameters
         ----------
         other
+        setitem : bool, default False
+            For __setitem__ we may have stricter compatiblity resrictions than
+            for comparisons.
 
         Raises
         ------
@@ -500,10 +579,10 @@ def __setitem__(
                     return
 
             value = type(self)._from_sequence(value, dtype=self.dtype)
-            self._check_compatible_with(value)
+            self._check_compatible_with(value, setitem=True)
             value = value.asi8
         elif isinstance(value, self._scalar_type):
-            self._check_compatible_with(value)
+            self._check_compatible_with(value, setitem=True)
             value = self._unbox_scalar(value)
         elif is_valid_nat_for_dtype(value, self.dtype):
             value = iNaT
@@ -588,7 +667,17 @@ def _validate_fill_value(self, fill_value):
         ------
         ValueError
         """
-        raise AbstractMethodError(self)
+        if isna(fill_value):
+            fill_value = iNaT
+        elif isinstance(fill_value, self._recognized_scalars):
+            self._check_compatible_with(fill_value)
+            fill_value = self._scalar_type(fill_value)
+            fill_value = self._unbox_scalar(fill_value)
+        else:
+            raise ValueError(
+                f"'fill_value' should be a {self._scalar_type}. Got '{fill_value}'."
+            )
+        return fill_value
 
     def take(self, indices, allow_fill=False, fill_value=None):
         if allow_fill:
@@ -921,6 +1010,7 @@ def _is_unique(self):
 
     # ------------------------------------------------------------------
     # Arithmetic Methods
+    _create_comparison_method = classmethod(_datetimelike_array_cmp)
 
     # pow is invalid for all three subclasses; TimedeltaArray will override
     #  the multiplication and division ops
@@ -1472,6 +1562,8 @@ def mean(self, skipna=True):
         return self._box_func(result)
 
 
+DatetimeLikeArrayMixin._add_comparison_ops()
+
 # -------------------------------------------------------------------
 # Shared Constructor Helpers