Merge branch 'main' into pandas.Index

Author: aBiR1D

ci/code_checks.sh

@@ -92,22 +92,17 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Grouper PR02" \
         -i "pandas.Index.append PR07,RT03,SA01" \
         -i "pandas.Index.difference PR07,RT03,SA01" \
-        -i "pandas.Index.duplicated RT03" \
         -i "pandas.Index.get_indexer PR07,SA01" \
         -i "pandas.Index.get_indexer_for PR01,SA01" \
         -i "pandas.Index.get_indexer_non_unique PR07,SA01" \
         -i "pandas.Index.get_loc PR07,RT03,SA01" \
-        -i "pandas.Index.identical PR01,SA01" \
         -i "pandas.Index.join PR07,RT03,SA01" \
         -i "pandas.Index.names GL08" \
-        -i "pandas.Index.nunique RT03" \
         -i "pandas.Index.putmask PR01,RT03" \
         -i "pandas.Index.ravel PR01,RT03" \
         -i "pandas.Index.slice_indexer PR07,RT03,SA01" \
         -i "pandas.Index.str PR01,SA01" \
-        -i "pandas.Index.symmetric_difference PR07,RT03,SA01" \
         -i "pandas.Index.take PR01,PR07" \
-        -i "pandas.Index.union PR07,RT03,SA01" \
         -i "pandas.Index.view GL08" \
         -i "pandas.Int16Dtype SA01" \
         -i "pandas.Int32Dtype SA01" \
@@ -257,7 +252,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.mode SA01" \
         -i "pandas.Series.mul PR07" \
         -i "pandas.Series.ne PR07,SA01" \
-        -i "pandas.Series.nunique RT03" \
         -i "pandas.Series.pad PR01,SA01" \
         -i "pandas.Series.plot PR02,SA01" \
         -i "pandas.Series.pop RT03,SA01" \

doc/source/whatsnew/v3.0.0.rst

@@ -221,6 +221,7 @@ Removal of prior version deprecations/changes
 - :meth:`SeriesGroupBy.agg` no longer pins the name of the group to the input passed to the provided ``func`` (:issue:`51703`)
 - All arguments except ``name`` in :meth:`Index.rename` are now keyword only (:issue:`56493`)
 - All arguments except the first ``path``-like argument in IO writers are now keyword only (:issue:`54229`)
+- Changed behavior of :meth:`Series.__getitem__` and :meth:`Series.__setitem__` to always treat integer keys as labels, never as positional, consistent with :class:`DataFrame` behavior (:issue:`50617`)
 - Disallow allowing logical operations (``||``, ``&``, ``^``) between pandas objects and dtype-less sequences (e.g. ``list``, ``tuple``); wrap the objects in :class:`Series`, :class:`Index`, or ``np.array`` first instead (:issue:`52264`)
 - Disallow automatic casting to object in :class:`Series` logical operations (``&``, ``^``, ``||``) between series with mismatched indexes and dtypes other than ``object`` or ``bool`` (:issue:`52538`)
 - Disallow calling :meth:`Series.replace` or :meth:`DataFrame.replace` without a ``value`` and with non-dict-like ``to_replace`` (:issue:`33302`)
@@ -389,6 +390,7 @@ Numeric
 
 Conversion
 ^^^^^^^^^^
+- Bug in :meth:`DataFrame.astype` not casting ``values`` for Arrow-based dictionary dtype correctly (:issue:`58479`)
 - Bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`)
 - Bug in :meth:`Series.astype` might modify read-only array inplace when casting to a string dtype (:issue:`57212`)
 - Bug in :meth:`Series.reindex` not maintaining ``float32`` type when a ``reindex`` introduces a missing value (:issue:`45857`)
@@ -469,6 +471,7 @@ Styler
 Other
 ^^^^^
 - Bug in :class:`DataFrame` when passing a ``dict`` with a NA scalar and ``columns`` that would always return ``np.nan`` (:issue:`57205`)
+- Bug in :func:`eval` where the names of the :class:`Series` were not preserved when using ``engine="numexpr"``. (:issue:`10239`)
 - Bug in :func:`unique` on :class:`Index` not always returning :class:`Index` (:issue:`57043`)
 - Bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using NumPy attributes via ``@`` notation, e.g., ``df.eval("@np.floor(a)")``. (:issue:`58041`)
 - Bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which did not allow to use ``tan`` function. (:issue:`55091`)
@@ -480,6 +483,7 @@ Other
 - Bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 - Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
 - Bug in Dataframe Interchange Protocol implementation was returning incorrect results for data buffers' associated dtype, for string and datetime columns (:issue:`54781`)
+- Bug in ``Series.list`` methods not preserving the original :class:`Index`. (:issue:`58425`)
 
 .. ***DO NOT USE THIS SECTION***
 

pandas/core/arrays/arrow/accessors.py

@@ -110,7 +110,9 @@ def len(self) -> Series:
         from pandas import Series
 
         value_lengths = pc.list_value_length(self._pa_array)
-        return Series(value_lengths, dtype=ArrowDtype(value_lengths.type))
+        return Series(
+            value_lengths, dtype=ArrowDtype(value_lengths.type), index=self._data.index
+        )
 
     def __getitem__(self, key: int | slice) -> Series:
         """
@@ -149,7 +151,9 @@ def __getitem__(self, key: int | slice) -> Series:
             # if key < 0:
             #     key = pc.add(key, pc.list_value_length(self._pa_array))
             element = pc.list_element(self._pa_array, key)
-            return Series(element, dtype=ArrowDtype(element.type))
+            return Series(
+                element, dtype=ArrowDtype(element.type), index=self._data.index
+            )
         elif isinstance(key, slice):
             if pa_version_under11p0:
                 raise NotImplementedError(
@@ -167,7 +171,7 @@ def __getitem__(self, key: int | slice) -> Series:
             if step is None:
                 step = 1
             sliced = pc.list_slice(self._pa_array, start, stop, step)
-            return Series(sliced, dtype=ArrowDtype(sliced.type))
+            return Series(sliced, dtype=ArrowDtype(sliced.type), index=self._data.index)
         else:
             raise ValueError(f"key must be an int or slice, got {type(key).__name__}")
 
@@ -195,15 +199,17 @@ def flatten(self) -> Series:
         ... )
         >>> s.list.flatten()
         0    1
-        1    2
-        2    3
-        3    3
+        0    2
+        0    3
+        1    3
         dtype: int64[pyarrow]
         """
         from pandas import Series
 
-        flattened = pc.list_flatten(self._pa_array)
-        return Series(flattened, dtype=ArrowDtype(flattened.type))
+        counts = pa.compute.list_value_length(self._pa_array)
+        flattened = pa.compute.list_flatten(self._pa_array)
+        index = self._data.index.repeat(counts.fill_null(pa.scalar(0, counts.type)))
+        return Series(flattened, dtype=ArrowDtype(flattened.type), index=index)
 
 
 class StructAccessor(ArrowAccessor):

pandas/core/arrays/arrow/array.py

@@ -525,6 +525,8 @@ def _box_pa_array(
         if pa_type is not None and pa_array.type != pa_type:
             if pa.types.is_dictionary(pa_type):
                 pa_array = pa_array.dictionary_encode()
+                if pa_array.type != pa_type:
+                    pa_array = pa_array.cast(pa_type)
             else:
                 try:
                     pa_array = pa_array.cast(pa_type)

pandas/core/base.py

@@ -1062,6 +1062,7 @@ def nunique(self, dropna: bool = True) -> int:
         Returns
         -------
         int
+            A integer indicating the number of unique elements in the object.
 
         See Also
         --------

pandas/core/computation/align.py

@@ -160,19 +160,24 @@ def align_terms(terms):
         # can't iterate so it must just be a constant or single variable
         if isinstance(terms.value, (ABCSeries, ABCDataFrame)):
             typ = type(terms.value)
-            return typ, _zip_axes_from_type(typ, terms.value.axes)
-        return np.result_type(terms.type), None
+            name = terms.value.name if isinstance(terms.value, ABCSeries) else None
+            return typ, _zip_axes_from_type(typ, terms.value.axes), name
+        return np.result_type(terms.type), None, None
 
     # if all resolved variables are numeric scalars
     if all(term.is_scalar for term in terms):
-        return result_type_many(*(term.value for term in terms)).type, None
+        return result_type_many(*(term.value for term in terms)).type, None, None
+
+    # if all input series have a common name, propagate it to the returned series
+    names = {term.value.name for term in terms if isinstance(term.value, ABCSeries)}
+    name = names.pop() if len(names) == 1 else None
 
     # perform the main alignment
     typ, axes = _align_core(terms)
-    return typ, axes
+    return typ, axes, name
 
 
-def reconstruct_object(typ, obj, axes, dtype):
+def reconstruct_object(typ, obj, axes, dtype, name):
     """
     Reconstruct an object given its type, raw value, and possibly empty
     (None) axes.
@@ -200,7 +205,9 @@ def reconstruct_object(typ, obj, axes, dtype):
     res_t = np.result_type(obj.dtype, dtype)
 
     if not isinstance(typ, partial) and issubclass(typ, PandasObject):
-        return typ(obj, dtype=res_t, **axes)
+        if name is None:
+            return typ(obj, dtype=res_t, **axes)
+        return typ(obj, dtype=res_t, name=name, **axes)
 
     # special case for pathological things like ~True/~False
     if hasattr(res_t, "type") and typ == np.bool_ and res_t != np.bool_:

pandas/core/computation/engines.py

@@ -54,6 +54,7 @@ def __init__(self, expr) -> None:
         self.expr = expr
         self.aligned_axes = None
         self.result_type = None
+        self.result_name = None
 
     def convert(self) -> str:
         """
@@ -76,12 +77,18 @@ def evaluate(self) -> object:
             The result of the passed expression.
         """
         if not self._is_aligned:
-            self.result_type, self.aligned_axes = align_terms(self.expr.terms)
+            self.result_type, self.aligned_axes, self.result_name = align_terms(
+                self.expr.terms
+            )
 
         # make sure no names in resolvers and locals/globals clash
         res = self._evaluate()
         return reconstruct_object(
-            self.result_type, res, self.aligned_axes, self.expr.terms.return_type
+            self.result_type,
+            res,
+            self.aligned_axes,
+            self.expr.terms.return_type,
+            self.result_name,
         )
 
     @property

pandas/core/indexes/base.py

@@ -2768,6 +2768,7 @@ def duplicated(self, keep: DropKeep = "first") -> npt.NDArray[np.bool_]:
         Returns
         -------
         np.ndarray[bool]
+            A numpy array of boolean values indicating duplicate index values.
 
         See Also
         --------
@@ -2873,6 +2874,8 @@ def union(self, other, sort=None):
         Parameters
         ----------
         other : Index or array-like
+            Index or an array-like object containing elements to form the union
+            with the original Index.
         sort : bool or None, default None
             Whether to sort the resulting Index.
 
@@ -2889,6 +2892,14 @@ def union(self, other, sort=None):
         Returns
         -------
         Index
+            Returns a new Index object with all unique elements from both the original
+            Index and the `other` Index.
+
+        See Also
+        --------
+        Index.unique : Return unique values in the index.
+        Index.intersection : Form the intersection of two Index objects.
+        Index.difference : Return a new Index with elements of index not in `other`.
 
         Examples
         --------
@@ -3313,7 +3324,10 @@ def symmetric_difference(self, other, result_name=None, sort=None):
         Parameters
         ----------
         other : Index or array-like
+            Index or an array-like object with elements to compute the symmetric
+            difference with the original Index.
         result_name : str
+            A string representing the name of the resulting Index, if desired.
         sort : bool or None, default None
             Whether to sort the resulting index. By default, the
             values are attempted to be sorted, but any TypeError from
@@ -3327,6 +3341,14 @@ def symmetric_difference(self, other, result_name=None, sort=None):
         Returns
         -------
         Index
+            Returns a new Index object containing elements that appear in either the
+            original Index or the `other` Index, but not both.
+
+        See Also
+        --------
+        Index.difference : Return a new Index with elements of index not in other.
+        Index.union : Form the union of two Index objects.
+        Index.intersection : Form the intersection of two Index objects.
 
         Notes
         -----
@@ -5323,12 +5345,23 @@ def identical(self, other) -> bool:
         """
         Similar to equals, but checks that object attributes and types are also equal.
 
+        Parameters
+        ----------
+        other : Index
+            The Index object you want to compare with the current Index object.
+
         Returns
         -------
         bool
             If two Index objects have equal elements and same type True,
             otherwise False.
 
+        See Also
+        --------
+        Index.equals: Determine if two Index object are equal.
+        Index.has_duplicates: Check if the Index has duplicate values.
+        Index.is_unique: Return if the index has unique values.
+
         Examples
         --------
         >>> idx1 = pd.Index(["1", "2", "3"])