DEPR: accepting Manager objects in DataFrame/Series (#52419)

doc/source/user_guide/10min.rst

@@ -763,12 +763,14 @@ Parquet
 Writing to a Parquet file:
 
 .. ipython:: python
+ :okwarning:
 
  df.to_parquet("foo.parquet")
 
 Reading from a Parquet file Store using :func:`read_parquet`:
 
 .. ipython:: python
+ :okwarning:
 
  pd.read_parquet("foo.parquet")
 

doc/source/user_guide/io.rst

@@ -2247,6 +2247,7 @@ For line-delimited json files, pandas can also return an iterator which reads in
 Line-limited json can also be read using the pyarrow reader by specifying ``engine="pyarrow"``.
 
 .. ipython:: python
+ :okwarning:
 
  from io import BytesIO
  df = pd.read_json(BytesIO(jsonl.encode()), lines=True, engine="pyarrow")
@@ -5554,6 +5555,7 @@ Read from an orc file.
 Read only certain columns of an orc file.
 
 .. ipython:: python
+ :okwarning:
 
  result = pd.read_orc(
  "example_pa.orc",

doc/source/user_guide/pyarrow.rst

@@ -104,6 +104,7 @@ To convert a :external+pyarrow:py:class:`pyarrow.Table` to a :class:`DataFrame`,
 :external+pyarrow:py:meth:`pyarrow.Table.to_pandas` method with ``types_mapper=pd.ArrowDtype``.
 
 .. ipython:: python
+ :okwarning:
 
  table = pa.table([pa.array([1, 2, 3], type=pa.int64())], names=["a"])
 
@@ -164,6 +165,7 @@ functions provide an ``engine`` keyword that can dispatch to PyArrow to accelera
 * :func:`read_feather`
 
 .. ipython:: python
+ :okwarning:
 
  import io
  data = io.StringIO("""a,b,c
@@ -178,6 +180,7 @@ PyArrow-backed data by specifying the parameter ``dtype_backend="pyarrow"``. A r
 ``engine="pyarrow"`` to necessarily return PyArrow-backed data.
 
 .. ipython:: python
+ :okwarning:
 
  import io
  data = io.StringIO("""a,b,c,d,e,f,g,h,i

doc/source/user_guide/scale.rst

@@ -51,6 +51,7 @@ To load the columns we want, we have two options.
 Option 1 loads in all the data and then filters to what we need.
 
 .. ipython:: python
+ :okwarning:
 
  columns = ["id_0", "name_0", "x_0", "y_0"]
 
@@ -59,6 +60,7 @@ Option 1 loads in all the data and then filters to what we need.
 Option 2 only loads the columns we request.
 
 .. ipython:: python
+ :okwarning:
 
  pd.read_parquet("timeseries_wide.parquet", columns=columns)
 
@@ -200,6 +202,7 @@ counts up to this point. As long as each individual file fits in memory, this wi
 work for arbitrary-sized datasets.
 
 .. ipython:: python
+ :okwarning:
 
  %%time
  files = pathlib.Path("data/timeseries/").glob("ts*.parquet")

doc/source/whatsnew/v2.0.0.rst

@@ -152,6 +152,7 @@ When this keyword is set to ``"pyarrow"``, then these functions will return pyar
 * :meth:`Series.convert_dtypes`
 
 .. ipython:: python
+ :okwarning:
 
  import io
  data = io.StringIO("""a,b,c,d,e,f,g,h,i

doc/source/whatsnew/v2.2.0.rst

@@ -249,6 +249,7 @@ Other Deprecations
 - Deprecated allowing non-keyword arguments in :meth:`DataFrame.to_pickle` except ``path``. (:issue:`54229`)
 - Deprecated allowing non-keyword arguments in :meth:`DataFrame.to_string` except ``buf``. (:issue:`54229`)
 - Deprecated allowing non-keyword arguments in :meth:`DataFrame.to_xml` except ``path_or_buffer``. (:issue:`54229`)
+- Deprecated allowing passing :class:`BlockManager` objects to :class:`DataFrame` or :class:`SingleBlockManager` objects to :class:`Series` (:issue:`52419`)
 - Deprecated automatic downcasting of object-dtype results in :meth:`Series.replace` and :meth:`DataFrame.replace`, explicitly call ``result = result.infer_objects(copy=False)`` instead. To opt in to the future version, use ``pd.set_option("future.no_silent_downcasting", True)`` (:issue:`54710`)
 - Deprecated downcasting behavior in :meth:`Series.where`, :meth:`DataFrame.where`, :meth:`Series.mask`, :meth:`DataFrame.mask`, :meth:`Series.clip`, :meth:`DataFrame.clip`; in a future version these will not infer object-dtype columns to non-object dtype, or all-round floats to integer dtype. Call ``result.infer_objects(copy=False)`` on the result for object inference, or explicitly cast floats to ints. To opt in to the future version, use ``pd.set_option("future.no_silent_downcasting", True)`` (:issue:`53656`)
 - Deprecated including the groups in computations when using :meth:`DataFrameGroupBy.apply` and :meth:`DataFrameGroupBy.resample`; pass ``include_groups=False`` to exclude the groups (:issue:`7155`)

pandas/conftest.py

@@ -178,6 +178,7 @@ def pytest_collection_modifyitems(items, config) -> None:
  "DataFrameGroupBy.fillna",
  "DataFrame.fillna with 'method' is deprecated",
  ),
+ ("read_parquet", "Passing a BlockManager to DataFrame is deprecated"),
  ]
 
  for item in items:

pandas/core/arraylike.py

@@ -263,7 +263,10 @@ def array_ufunc(self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any)
  Series,
  )
  from pandas.core.generic import NDFrame
- from pandas.core.internals import BlockManager
+ from pandas.core.internals import (
+ ArrayManager,
+ BlockManager,
+ )
 
  cls = type(self)
 
@@ -347,7 +350,7 @@ def _reconstruct(result):
  if method == "outer":
  raise NotImplementedError
  return result
- if isinstance(result, BlockManager):
+ if isinstance(result, (BlockManager, ArrayManager)):
  # we went through BlockManager.apply e.g. np.sqrt
  result = self._constructor_from_mgr(result, axes=result.axes)
  else:

pandas/core/frame.py

@@ -644,7 +644,6 @@ def _constructor(self) -> Callable[..., DataFrame]:
 
  def _constructor_from_mgr(self, mgr, axes):
  df = self._from_mgr(mgr, axes=axes)
-
  if type(self) is DataFrame:
  # fastpath avoiding constructor call
  return df
@@ -677,17 +676,29 @@ def __init__(
  dtype: Dtype | None = None,
  copy: bool | None = None,
  ) -> None:
+ allow_mgr = False
  if dtype is not None:
  dtype = self._validate_dtype(dtype)
 
  if isinstance(data, DataFrame):
  data = data._mgr
+ allow_mgr = True
  if not copy:
  # if not copying data, ensure to still return a shallow copy
  # to avoid the result sharing the same Manager
  data = data.copy(deep=False)
 
  if isinstance(data, (BlockManager, ArrayManager)):
+ if not allow_mgr:
+ # GH#52419
+ warnings.warn(
+ f"Passing a {type(data).__name__} to {type(self).__name__} "
+ "is deprecated and will raise in a future version. "
+ "Use public APIs instead.",
+ DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
+
  if using_copy_on_write():
  data = data.copy(deep=False)
  # first check if a Manager is passed without any other arguments
@@ -2462,7 +2473,7 @@ def maybe_reorder(
  manager = _get_option("mode.data_manager", silent=True)
  mgr = arrays_to_mgr(arrays, columns, result_index, typ=manager)
 
- return cls(mgr)
+ return cls._from_mgr(mgr, axes=mgr.axes)
 
  def to_records(
  self, index: bool = True, column_dtypes=None, index_dtypes=None
@@ -2672,7 +2683,7 @@ def _from_arrays(
  verify_integrity=verify_integrity,
  typ=manager,
  )
- return cls(mgr)
+ return cls._from_mgr(mgr, axes=mgr.axes)
 
  @doc(
  storage_options=_shared_docs["storage_options"],

pandas/core/generic.py

@@ -829,7 +829,8 @@ def swapaxes(self, axis1: Axis, axis2: Axis, copy: bool_t | None = None) -> Self
  if not using_copy_on_write() and copy is not False:
  new_mgr = new_mgr.copy(deep=True)
 
- return self._constructor(new_mgr).__finalize__(self, method="swapaxes")
+ out = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
+ return out.__finalize__(self, method="swapaxes")
 
  return self._constructor(
  new_values,

pandas/core/series.py

@@ -390,12 +390,22 @@ def __init__(
  else:
  fastpath = False
 
+ allow_mgr = False
  if (
  isinstance(data, (SingleBlockManager, SingleArrayManager))
  and index is None
  and dtype is None
  and (copy is False or copy is None)
  ):
+ if not allow_mgr:
+ # GH#52419
+ warnings.warn(
+ f"Passing a {type(data).__name__} to {type(self).__name__} "
+ "is deprecated and will raise in a future version. "
+ "Use public APIs instead.",
+ DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
  if using_copy_on_write():
  data = data.copy(deep=False)
  # GH#33357 called with just the SingleBlockManager
@@ -423,8 +433,19 @@ def __init__(
  data = SingleBlockManager.from_array(data, index)
  elif manager == "array":
  data = SingleArrayManager.from_array(data, index)
+ allow_mgr = True
  elif using_copy_on_write() and not copy:
  data = data.copy(deep=False)
+
+ if not allow_mgr:
+ warnings.warn(
+ f"Passing a {type(data).__name__} to {type(self).__name__} "
+ "is deprecated and will raise in a future version. "
+ "Use public APIs instead.",
+ DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
+
  if copy:
  data = data.copy()
  # skips validation of the name
@@ -435,6 +456,15 @@ def __init__(
  if isinstance(data, SingleBlockManager) and using_copy_on_write() and not copy:
  data = data.copy(deep=False)
 
+ if not allow_mgr:
+ warnings.warn(
+ f"Passing a {type(data).__name__} to {type(self).__name__} "
+ "is deprecated and will raise in a future version. "
+ "Use public APIs instead.",
+ DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
+
  name = ibase.maybe_extract_name(name, data, type(self))
 
  if index is not None:
@@ -500,6 +530,16 @@ def __init__(
  "`index` argument. `copy` must be False."
  )
 
+ if not allow_mgr:
+ warnings.warn(
+ f"Passing a {type(data).__name__} to {type(self).__name__} "
+ "is deprecated and will raise in a future version. "
+ "Use public APIs instead.",
+ DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
+ allow_mgr = True
+
  elif isinstance(data, ExtensionArray):
  pass
  else:
@@ -612,22 +652,14 @@ def _constructor_expanddim(self) -> Callable[..., DataFrame]:
  return DataFrame
 
  def _expanddim_from_mgr(self, mgr, axes) -> DataFrame:
- # https://github.com/pandas-dev/pandas/pull/52132#issuecomment-1481491828
- # This is a short-term implementation that will be replaced
- # with self._constructor_expanddim._constructor_from_mgr(...)
- # once downstream packages (geopandas) have had a chance to implement
- # their own overrides.
- # error: "Callable[..., DataFrame]" has no attribute "_from_mgr" [attr-defined]
- from pandas import DataFrame
+ from pandas.core.frame import DataFrame
 
  return DataFrame._from_mgr(mgr, axes=mgr.axes)
 
  def _constructor_expanddim_from_mgr(self, mgr, axes):
  df = self._expanddim_from_mgr(mgr, axes)
  if type(self) is Series:
- # fastpath avoiding constructor
  return df
- assert axes is mgr.axes
  return self._constructor_expanddim(df, copy=False)
 
  # types

pandas/tests/arrays/interval/test_interval.py

@@ -337,20 +337,26 @@ def test_arrow_table_roundtrip(breaks):
 
  table = pa.table(df)
  assert isinstance(table.field("a").type, ArrowIntervalType)
- result = table.to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.to_pandas()
  assert isinstance(result["a"].dtype, pd.IntervalDtype)
  tm.assert_frame_equal(result, df)
 
  table2 = pa.concat_tables([table, table])
- result = table2.to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table2.to_pandas()
  expected = pd.concat([df, df], ignore_index=True)
  tm.assert_frame_equal(result, expected)
 
  # GH-41040
  table = pa.table(
  [pa.chunked_array([], type=table.column(0).type)], schema=table.schema
  )
- result = table.to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.to_pandas()
  tm.assert_frame_equal(result, expected[0:0])
 
 
@@ -371,7 +377,9 @@ def test_arrow_table_roundtrip_without_metadata(breaks):
  table = table.replace_schema_metadata()
  assert table.schema.metadata is None
 
- result = table.to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.to_pandas()
  assert isinstance(result["a"].dtype, pd.IntervalDtype)
  tm.assert_frame_equal(result, df)
 

pandas/tests/arrays/masked/test_arrow_compat.py

@@ -35,7 +35,10 @@ def test_arrow_roundtrip(data):
  df = pd.DataFrame({"a": data})
  table = pa.table(df)
  assert table.field("a").type == str(data.dtype.numpy_dtype)
- result = table.to_pandas()
+
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.to_pandas()
  assert result["a"].dtype == data.dtype
  tm.assert_frame_equal(result, df)
 
@@ -53,7 +56,9 @@ def types_mapper(arrow_type):
  record_batch = pa.RecordBatch.from_arrays(
  [bools_array, ints_array, small_ints_array], ["bools", "ints", "small_ints"]
  )
- result = record_batch.to_pandas(types_mapper=types_mapper)
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = record_batch.to_pandas(types_mapper=types_mapper)
  bools = pd.Series([True, None, False], dtype="boolean")
  ints = pd.Series([1, None, 2], dtype="Int64")
  small_ints = pd.Series([-1, 0, 7], dtype="Int64")
@@ -70,7 +75,9 @@ def test_arrow_load_from_zero_chunks(data):
  table = pa.table(
  [pa.chunked_array([], type=table.field("a").type)], schema=table.schema
  )
- result = table.to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.to_pandas()
  assert result["a"].dtype == data.dtype
  tm.assert_frame_equal(result, df)
 
@@ -91,14 +98,18 @@ def test_arrow_sliced(data):
 
  df = pd.DataFrame({"a": data})
  table = pa.table(df)
- result = table.slice(2, None).to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.slice(2, None).to_pandas()
  expected = df.iloc[2:].reset_index(drop=True)
  tm.assert_frame_equal(result, expected)
 
  # no missing values
  df2 = df.fillna(data[0])
  table = pa.table(df2)
- result = table.slice(2, None).to_pandas()
+ msg = "Passing a BlockManager to DataFrame is deprecated"
+ with tm.assert_produces_warning(DeprecationWarning, match=msg):
+ result = table.slice(2, None).to_pandas()
  expected = df2.iloc[2:].reset_index(drop=True)
  tm.assert_frame_equal(result, expected)