Merge branch 'pandas-dev:main' into main

Author: Kyrpel

doc/source/getting_started/install.rst

@@ -276,7 +276,7 @@ Computation
 ========================= ================== =============================================================
 Dependency                Minimum Version    Notes
 ========================= ================== =============================================================
-SciPy                     1.14.1             Miscellaneous statistical functions
+SciPy                     1.4.1              Miscellaneous statistical functions
 numba                     0.50.1             Alternative execution engine for rolling operations
                                              (see :ref:`Enhancing Performance <enhancingperf.numba>`)
 xarray                    0.15.1             pandas-like API for N-dimensional data

doc/source/user_guide/io.rst

@@ -186,6 +186,11 @@ dtype : Type name or dict of column -> type, default ``None``
   (unsupported with ``engine='python'``). Use ``str`` or ``object`` together
   with suitable ``na_values`` settings to preserve and
   not interpret dtype.
+  .. versionadded:: 1.5.0
+
+    Support for defaultdict was added. Specify a defaultdict as input where
+    the default determines the dtype of the columns which are not explicitly
+    listed.
 engine : {``'c'``, ``'python'``, ``'pyarrow'``}
   Parser engine to use. The C and pyarrow engines are faster, while the python engine
   is currently more feature-complete. Multithreading is currently only supported by

doc/source/whatsnew/v1.5.0.rst

@@ -39,6 +39,7 @@ Other enhancements
 - :meth:`to_numeric` now preserves float64 arrays when downcasting would generate values not representable in float32 (:issue:`43693`)
 - :meth:`Series.reset_index` and :meth:`DataFrame.reset_index` now support the argument ``allow_duplicates`` (:issue:`44410`)
 - :meth:`.GroupBy.min` and :meth:`.GroupBy.max` now supports `Numba <https://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`45428`)
+- :func:`read_csv` now supports ``defaultdict`` as a ``dtype`` parameter (:issue:`41574`)
 - :meth:`DataFrame.rolling` and :meth:`Series.rolling` now support a ``step`` parameter with fixed-length windows (:issue:`15354`)
 - Implemented a ``bool``-dtype :class:`Index`, passing a bool-dtype array-like to ``pd.Index`` will now retain ``bool`` dtype instead of casting to ``object`` (:issue:`45061`)
 - Implemented a complex-dtype :class:`Index`, passing a complex-dtype array-like to ``pd.Index`` will now retain complex dtype instead of casting to ``object`` (:issue:`45845`)
@@ -83,32 +84,51 @@ did not have the same index as the input.
 
 .. code-block:: ipython
 
-    In [3]: df.groupby('a', dropna=True).transform(lambda x: x.sum())
+    In [3]: # Value in the last row should be np.nan
+            df.groupby('a', dropna=True).transform('sum')
     Out[3]:
        b
     0  5
     1  5
+    2  5
 
-    In [3]: df.groupby('a', dropna=True).transform(lambda x: x)
+    In [3]: # Should have one additional row with the value np.nan
+            df.groupby('a', dropna=True).transform(lambda x: x.sum())
     Out[3]:
        b
-    0  2
-    1  3
+    0  5
+    1  5
 
-    In [3]: df.groupby('a', dropna=True).transform('sum')
+    In [3]: # The value in the last row is np.nan interpreted as an integer
+            df.groupby('a', dropna=True).transform('ffill')
+    Out[3]:
+                         b
+    0                    2
+    1                    3
+    2 -9223372036854775808
+
+    In [3]: # Should have one additional row with the value np.nan
+            df.groupby('a', dropna=True).transform(lambda x: x)
     Out[3]:
        b
-    0  5
-    1  5
-    2  5
+    0  2
+    1  3
 
 *New behavior*:
 
 .. ipython:: python
 
+    df.groupby('a', dropna=True).transform('sum')
     df.groupby('a', dropna=True).transform(lambda x: x.sum())
+    df.groupby('a', dropna=True).transform('ffill')
     df.groupby('a', dropna=True).transform(lambda x: x)
-    df.groupby('a', dropna=True).transform('sum')
+
+.. _whatsnew_150.notable_bug_fixes.visualization:
+
+Styler
+^^^^^^
+
+- Fix showing "None" as ylabel in :meth:`Series.plot` when not setting ylabel (:issue:`46129`)
 
 .. _whatsnew_150.notable_bug_fixes.notable_bug_fix2:
 
@@ -343,6 +363,7 @@ Other Deprecations
 - Deprecated treating all-bool ``object``-dtype columns as bool-like in :meth:`DataFrame.any` and :meth:`DataFrame.all` with ``bool_only=True``, explicitly cast to bool instead (:issue:`46188`)
 - Deprecated behavior of method :meth:`DataFrame.quantile`, attribute ``numeric_only`` will default False. Including datetime/timedelta columns in the result (:issue:`7308`).
 - Deprecated :attr:`Timedelta.freq` and :attr:`Timedelta.is_populated` (:issue:`46430`)
+- Deprecated :attr:`Timedelta.delta` (:issue:`46476`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -384,6 +405,7 @@ Datetimelike
 - Bug in :class:`Timestamp` construction when passing datetime components as positional arguments and ``tzinfo`` as a keyword argument incorrectly raising (:issue:`31929`)
 - Bug in :meth:`Index.astype` when casting from object dtype to ``timedelta64[ns]`` dtype incorrectly casting ``np.datetime64("NaT")`` values to ``np.timedelta64("NaT")`` instead of raising (:issue:`45722`)
 - Bug in :meth:`SeriesGroupBy.value_counts` index when passing categorical column (:issue:`44324`)
+- Bug in :meth:`DatetimeIndex.tz_localize` localizing to UTC failing to make a copy of the underlying data (:issue:`46460`)
 -
 
 Timedelta

pandas/_libs/interval.pyi

@@ -32,8 +32,6 @@ class _LengthDescriptor:
     def __get__(
         self, instance: Interval[_OrderableTimesT], owner: Any
     ) -> Timedelta: ...
-    @overload
-    def __get__(self, instance: IntervalTree, owner: Any) -> np.ndarray: ...
 
 class _MidDescriptor:
     @overload
@@ -42,8 +40,6 @@ class _MidDescriptor:
     def __get__(
         self, instance: Interval[_OrderableTimesT], owner: Any
     ) -> _OrderableTimesT: ...
-    @overload
-    def __get__(self, instance: IntervalTree, owner: Any) -> np.ndarray: ...
 
 class IntervalMixin:
     @property
@@ -54,8 +50,6 @@ class IntervalMixin:
     def open_left(self) -> bool: ...
     @property
     def open_right(self) -> bool: ...
-    mid: _MidDescriptor
-    length: _LengthDescriptor
     @property
     def is_empty(self) -> bool: ...
     def _check_closed_matches(self, other: IntervalMixin, name: str = ...) -> None: ...
@@ -67,6 +61,8 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
     def right(self: Interval[_OrderableT]) -> _OrderableT: ...
     @property
     def closed(self) -> IntervalClosedType: ...
+    mid: _MidDescriptor
+    length: _LengthDescriptor
     def __init__(
         self,
         left: _OrderableT,
@@ -162,6 +158,10 @@ class IntervalTree(IntervalMixin):
         closed: IntervalClosedType = ...,
         leaf_size: int = ...,
     ): ...
+    @property
+    def mid(self) -> np.ndarray: ...
+    @property
+    def length(self) -> np.ndarray: ...
     def get_indexer(self, target) -> npt.NDArray[np.intp]: ...
     def get_indexer_non_unique(
         self, target

pandas/_libs/parsers.pyx

@@ -1,6 +1,7 @@
 # Copyright (c) 2012, Lambda Foundry, Inc.
 # See LICENSE for the license
 from base64 import decode
+from collections import defaultdict
 from csv import (
     QUOTE_MINIMAL,
     QUOTE_NONE,
@@ -964,6 +965,8 @@ cdef class TextReader:
 
         results = {}
         nused = 0
+        is_default_dict_dtype = isinstance(self.dtype, defaultdict)
+
         for i in range(self.table_width):
             if i < self.leading_cols:
                 # Pass through leading columns always
@@ -994,6 +997,8 @@ cdef class TextReader:
                         col_dtype = self.dtype[name]
                     elif i in self.dtype:
                         col_dtype = self.dtype[i]
+                    elif is_default_dict_dtype:
+                        col_dtype = self.dtype[name]
                 else:
                     if self.dtype.names:
                         # structured array

pandas/_libs/tslibs/dtypes.pxd

@@ -1,8 +1,11 @@
+from numpy cimport int64_t
+
 from pandas._libs.tslibs.np_datetime cimport NPY_DATETIMEUNIT
 
 
 cdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit)
 cdef NPY_DATETIMEUNIT freq_group_code_to_npy_unit(int freq) nogil
+cdef int64_t periods_per_day(NPY_DATETIMEUNIT reso=*)
 
 cdef dict attrname_to_abbrevs
 

pandas/_libs/tslibs/dtypes.pyx

@@ -307,6 +307,36 @@ cdef NPY_DATETIMEUNIT freq_group_code_to_npy_unit(int freq) nogil:
         return NPY_DATETIMEUNIT.NPY_FR_D
 
 
+cdef int64_t periods_per_day(NPY_DATETIMEUNIT reso=NPY_DATETIMEUNIT.NPY_FR_ns):
+    """
+    How many of the given time units fit into a single day?
+    """
+    cdef:
+        int64_t day_units
+
+    if reso == NPY_DATETIMEUNIT.NPY_FR_ps:
+        # pico is the smallest unit for which we don't overflow, so
+        #  we exclude fempto and atto
+        day_units = 24 * 3600 * 1_000_000_000_000
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_ns:
+        day_units = 24 * 3600 * 1_000_000_000
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_us:
+        day_units = 24 * 3600 * 1_000_000
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_ms:
+        day_units = 24 * 3600 * 1_000
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_s:
+        day_units = 24 * 3600
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_m:
+        day_units = 24 * 60
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_h:
+        day_units = 24
+    elif reso == NPY_DATETIMEUNIT.NPY_FR_D:
+        day_units = 1
+    else:
+        raise NotImplementedError(reso)
+    return day_units
+
+
 cdef dict _reso_str_map = {
     Resolution.RESO_NS.value: "nanosecond",
     Resolution.RESO_US.value: "microsecond",

pandas/_libs/tslibs/timedeltas.pyx

@@ -1014,6 +1014,12 @@ cdef class _Timedelta(timedelta):
         >>> td.delta
         42
         """
+        # Deprecated GH#46476
+        warnings.warn(
+            "Timedelta.delta is deprecated and will be removed in a future version.",
+            FutureWarning,
+            stacklevel=1,
+        )
         return self.value
 
     @property

pandas/_libs/tslibs/tzconversion.pyx

@@ -64,8 +64,6 @@ cdef int64_t tz_localize_to_utc_single(
         return val - _tz_localize_using_tzinfo_api(val, tz, to_utc=True)
 
     elif is_fixed_offset(tz):
-        # TODO: in this case we should be able to use get_utcoffset,
-        #  that returns None for e.g. 'dateutil//usr/share/zoneinfo/Etc/GMT-9'
         _, deltas, _ = get_dst_info(tz)
         delta = deltas[0]
         return val - delta
@@ -121,9 +119,10 @@ timedelta-like}
         Py_ssize_t delta_idx_offset, delta_idx, pos_left, pos_right
         int64_t *tdata
         int64_t v, left, right, val, v_left, v_right, new_local, remaining_mins
-        int64_t first_delta
+        int64_t first_delta, delta
         int64_t shift_delta = 0
-        ndarray[int64_t] trans, result, result_a, result_b, dst_hours
+        ndarray[int64_t] trans, result_a, result_b, dst_hours
+        int64_t[::1] result
         npy_datetimestruct dts
         bint infer_dst = False, is_dst = False, fill = False
         bint shift_forward = False, shift_backward = False
@@ -132,7 +131,7 @@ timedelta-like}
 
     # Vectorized version of DstTzInfo.localize
     if is_utc(tz) or tz is None:
-        return vals
+        return vals.copy()
 
     result = np.empty(n, dtype=np.int64)
 
@@ -143,7 +142,18 @@ timedelta-like}
                 result[i] = NPY_NAT
             else:
                 result[i] = v - _tz_localize_using_tzinfo_api(v, tz, to_utc=True)
-        return result
+        return result.base  # to return underlying ndarray
+
+    elif is_fixed_offset(tz):
+        _, deltas, _ = get_dst_info(tz)
+        delta = deltas[0]
+        for i in range(n):
+            v = vals[i]
+            if v == NPY_NAT:
+                result[i] = NPY_NAT
+            else:
+                result[i] = v - delta
+        return result.base  # to return underlying ndarray
 
     # silence false-positive compiler warning
     ambiguous_array = np.empty(0, dtype=bool)
@@ -298,7 +308,7 @@ timedelta-like}
                 stamp = _render_tstamp(val)
                 raise pytz.NonExistentTimeError(stamp)
 
-    return result
+    return result.base  # .base to get underlying ndarray
 
 
 cdef inline Py_ssize_t bisect_right_i8(int64_t *data,

pandas/_libs/tslibs/vectorized.pyx

@@ -21,7 +21,9 @@ cnp.import_array()
 from .conversion cimport normalize_i8_stamp
 
 from .dtypes import Resolution
+
 from .ccalendar cimport DAY_NANOS
+from .dtypes cimport c_Resolution
 from .nattype cimport (
     NPY_NAT,
     c_NaT as NaT,
@@ -168,27 +170,19 @@ def ints_to_pydatetime(
 
 # -------------------------------------------------------------------------
 
-cdef:
-    int RESO_US = Resolution.RESO_US.value
-    int RESO_MS = Resolution.RESO_MS.value
-    int RESO_SEC = Resolution.RESO_SEC.value
-    int RESO_MIN = Resolution.RESO_MIN.value
-    int RESO_HR = Resolution.RESO_HR.value
-    int RESO_DAY = Resolution.RESO_DAY.value
-
 
-cdef inline int _reso_stamp(npy_datetimestruct *dts):
+cdef inline c_Resolution _reso_stamp(npy_datetimestruct *dts):
     if dts.us != 0:
         if dts.us % 1000 == 0:
-            return RESO_MS
-        return RESO_US
+            return c_Resolution.RESO_MS
+        return c_Resolution.RESO_US
     elif dts.sec != 0:
-        return RESO_SEC
+        return c_Resolution.RESO_SEC
     elif dts.min != 0:
-        return RESO_MIN
+        return c_Resolution.RESO_MIN
     elif dts.hour != 0:
-        return RESO_HR
-    return RESO_DAY
+        return c_Resolution.RESO_HR
+    return c_Resolution.RESO_DAY
 
 
 @cython.wraparound(False)
@@ -205,7 +199,7 @@ def get_resolution(const int64_t[:] stamps, tzinfo tz=None) -> Resolution:
         str typ
 
         npy_datetimestruct dts
-        int reso = RESO_DAY, curr_reso
+        c_Resolution reso = c_Resolution.RESO_DAY, curr_reso
 
     if is_utc(tz) or tz is None:
         use_utc = True

pandas/_typing.py

@@ -297,6 +297,9 @@ def closed(self) -> bool:
 else:
     TakeIndexer = Any
 
+# Shared by functions such as drop and astype
+IgnoreRaise = Literal["ignore", "raise"]
+
 # Windowing rank methods
 WindowingRankType = Literal["average", "min", "max"]
 
@@ -311,7 +314,7 @@ def closed(self) -> bool:
 
 # datetime and NaTType
 DatetimeNaTType = Union[datetime, "NaTType"]
-DateTimeErrorChoices = Literal["ignore", "raise", "coerce"]
+DateTimeErrorChoices = Union[IgnoreRaise, Literal["coerce"]]
 
 # sort_index
 SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"]

pandas/core/arrays/datetimes.py

@@ -456,9 +456,13 @@ def _generate_range(
             endpoint_tz = start.tz if start is not None else end.tz
 
             if tz is not None and endpoint_tz is None:
-                i8values = tzconversion.tz_localize_to_utc(
-                    i8values, tz, ambiguous=ambiguous, nonexistent=nonexistent
-                )
+
+                if not timezones.is_utc(tz):
+                    # short-circuit tz_localize_to_utc which would make
+                    #  an unnecessary copy with UTC but be a no-op.
+                    i8values = tzconversion.tz_localize_to_utc(
+                        i8values, tz, ambiguous=ambiguous, nonexistent=nonexistent
+                    )
 
                 # i8values is localized datetime64 array -> have to convert
                 # start/end as well to compare
@@ -2126,6 +2130,8 @@ def _sequence_to_dt64ns(
         if tz is not None:
             # Convert tz-naive to UTC
             tz = timezones.maybe_get_tz(tz)
+            # TODO: if tz is UTC, are there situations where we *don't* want a
+            #  copy?  tz_localize_to_utc always makes one.
             data = tzconversion.tz_localize_to_utc(
                 data.view("i8"), tz, ambiguous=ambiguous
             )