[nomerge] Amend _from_array functionality to _from_arrays

Author: irgolic

pandas/core/internals/construction.py

@@ -27,6 +27,7 @@
     dict_compat,
     maybe_cast_to_datetime,
     maybe_convert_platform,
+    maybe_infer_to_datetimelike,
     maybe_upcast,
 )
 from pandas.core.dtypes.common import (
@@ -36,6 +37,7 @@
     is_integer_dtype,
     is_list_like,
     is_named_tuple,
+    is_object_dtype,
 )
 from pandas.core.dtypes.generic import (
     ABCDataFrame,
@@ -55,10 +57,7 @@
     get_objs_combined_axis,
     union_indexes,
 )
-from pandas.core.internals.managers import (
-    create_block_manager_from_array,
-    create_block_manager_from_arrays,
-)
+from pandas.core.internals.managers import create_block_manager_from_arrays
 
 if TYPE_CHECKING:
     from numpy.ma.mrecords import MaskedRecords
@@ -230,7 +229,23 @@ def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
     )
     values = values.T
 
-    return create_block_manager_from_array(values, [columns, index], dtype)
+    # if we don't have a dtype specified, then try to convert objects
+    # on the entire block; this is to convert if we have datetimelike's
+    # embedded in an object type
+    if dtype is None and is_object_dtype(values.dtype):
+        maybe_datetime = [maybe_infer_to_datetimelike(instance) for instance in values]
+        # don't convert (and copy) the objects if no type inference occurs
+        if any(
+            not is_dtype_equal(instance.dtype, values.dtype)
+            for instance in maybe_datetime
+        ):
+            arrays = maybe_datetime
+        else:
+            arrays = [values]
+    else:
+        arrays = [values]
+
+    return create_block_manager_from_arrays(arrays, columns, [columns, index])
 
 
 def init_dict(data: Dict, index, columns, dtype: Optional[DtypeObj] = None):

pandas/core/internals/managers.py

@@ -27,15 +27,13 @@
 from pandas.core.dtypes.cast import (
     find_common_type,
     infer_dtype_from_scalar,
-    maybe_infer_to_datetimelike,
     maybe_promote,
 )
 from pandas.core.dtypes.common import (
     DT64NS_DTYPE,
     is_dtype_equal,
     is_extension_array_dtype,
     is_list_like,
-    is_object_dtype,
 )
 from pandas.core.dtypes.concat import concat_compat
 from pandas.core.dtypes.dtypes import ExtensionDtype
@@ -328,7 +326,7 @@ def _verify_integrity(self) -> None:
             if block.shape[1:] != mgr_shape[1:]:
                 raise construction_error(tot_items, block.shape[1:], self.axes)
         if len(self.items) != tot_items:
-            raise AssertionError(
+            raise ValueError(
                 "Number of manager items must equal union of "
                 f"block items\n# manager items: {len(self.items)}, # "
                 f"tot_items: {tot_items}"
@@ -1671,48 +1669,14 @@ def create_block_manager_from_arrays(
     # ensure we dont have any PandasArrays when we call get_block_type
     # Note: just calling extract_array breaks tests that patch PandasArray._typ.
     arrays = [x if not isinstance(x, ABCPandasArray) else x.to_numpy() for x in arrays]
+    blocks = _form_blocks(arrays, names, axes)
     try:
-        blocks = _form_blocks(arrays, names, axes)
         mgr = BlockManager(blocks, axes)
         mgr._consolidate_inplace()
         return mgr
     except ValueError as e:
-        raise construction_error(len(arrays), arrays[0].shape, axes, e)
-
-
-def create_block_manager_from_array(
-    array, axes: List[Index], dtype: Optional[Dtype] = None
-) -> BlockManager:
-    assert isinstance(axes, list)
-    assert all(isinstance(x, Index) for x in axes)
-
-    # ensure we dont have any PandasArrays when we call get_block_type
-    # Note: just calling extract_array breaks tests that patch PandasArray._typ.
-    array = array if not isinstance(array, ABCPandasArray) else array.to_numpy()
-
-    try:
-        # if we don't have a dtype specified, then try to convert objects
-        # on the entire block; this is to convert if we have datetimelike's
-        # embedded in an object type
-        if dtype is None and is_object_dtype(array.dtype):
-            maybe_datetime = [
-                maybe_infer_to_datetimelike(instance) for instance in array
-            ]
-            # don't convert (and copy) the objects if no type conversion occurs
-            if any(
-                not is_dtype_equal(instance.dtype, array.dtype)
-                for instance in maybe_datetime
-            ):
-                blocks = _form_blocks(maybe_datetime, axes[0], axes)
-            else:
-                blocks = [make_block(array, slice(0, len(axes[0])))]
-        else:
-            blocks = [make_block(array, slice(0, len(axes[0])), dtype=dtype)]
-        mgr = BlockManager(blocks, axes)
-        mgr._consolidate_inplace()
-        return mgr
-    except ValueError as e:
-        raise construction_error(array.shape[0], array.shape[1:], axes, e)
+        tot_items = sum(b.shape[0] for b in blocks)
+        raise construction_error(tot_items, blocks[0].shape[1:], axes, e)
 
 
 def construction_error(tot_items, block_shape, axes, e=None):
@@ -1739,36 +1703,60 @@ def construction_error(tot_items, block_shape, axes, e=None):
 # -----------------------------------------------------------------------
 
 
-def _form_blocks(arrays, names: Index, axes) -> List[Block]:
-    # put "leftover" items in float bucket, where else?
-    # generalize?
-
-    if len(arrays) != len(names):
-        raise ValueError(
-            f"Number of arrays ({len(arrays)}) "
-            f"does not match index length ({len(names)})"
-        )
-
+def _get_block_ranges(arrays, names, axes):
     items_dict: DefaultDict[str, List] = defaultdict(list)
     extra_locs = []
 
-    names_idx = names
-    if names_idx.equals(axes[0]):
-        names_indexer = np.arange(len(names_idx))
+    # GH#39263
+    # if one array is passed, form one contiguous block
+    if len(arrays) == 1:
+        first = arrays[0]
+        block_type = get_block_type(first)
+
+        if first.ndim == 1:
+            end = 1
+        else:
+            end = len(first)
+
+        items_dict[block_type.__name__].append(((0, end), first))
+    # else, form blocks column-by-column
     else:
-        assert names_idx.intersection(axes[0]).is_unique
-        names_indexer = names_idx.get_indexer_for(axes[0])
+        names_idx = names
+        if names_idx.equals(axes[0]):
+            names_indexer = np.arange(len(names_idx))
+        else:
+            assert names_idx.intersection(axes[0]).is_unique
+            names_indexer = names_idx.get_indexer_for(axes[0])
+
+        i = 0
+        for name_idx in names_indexer:
+            if name_idx == -1:
+                extra_locs.append(i)
+                i += 1
+                continue
+
+            v = arrays[name_idx]
+            block_type = get_block_type(v)
 
-    for i, name_idx in enumerate(names_indexer):
-        if name_idx == -1:
-            extra_locs.append(i)
-            continue
+            if v.ndim == 1:
+                ei = i + 1
+            else:
+                ei = i + v.shape[1]
+
+            items_dict[block_type.__name__].append(((i, ei), v))
+            i = ei
 
-        k = names[name_idx]
-        v = arrays[name_idx]
+    return items_dict, extra_locs
 
-        block_type = get_block_type(v)
-        items_dict[block_type.__name__].append((i, k, v))
+
+def _form_blocks(arrays, names: Index, axes) -> List[Block]:
+    # put "leftover" items in float bucket, where else?
+    # generalize?
+
+    items_dict: DefaultDict[str, List] = defaultdict(list)
+    extra_locs = []
+
+    items_dict, extra_locs = _get_block_ranges(arrays, names, axes)
 
     blocks: List[Block] = []
     if len(items_dict["FloatBlock"]):
@@ -1789,8 +1777,8 @@ def _form_blocks(arrays, names: Index, axes) -> List[Block]:
 
     if len(items_dict["DatetimeTZBlock"]):
         dttz_blocks = [
-            make_block(array, klass=DatetimeTZBlock, placement=i, ndim=2)
-            for i, _, array in items_dict["DatetimeTZBlock"]
+            make_block(array, klass=DatetimeTZBlock, placement=slice(*i), ndim=2)
+            for i, array in items_dict["DatetimeTZBlock"]
         ]
         blocks.extend(dttz_blocks)
 
@@ -1800,23 +1788,25 @@ def _form_blocks(arrays, names: Index, axes) -> List[Block]:
 
     if len(items_dict["CategoricalBlock"]) > 0:
         cat_blocks = [
-            make_block(array, klass=CategoricalBlock, placement=i, ndim=2)
-            for i, _, array in items_dict["CategoricalBlock"]
+            make_block(array, klass=CategoricalBlock, placement=slice(*i), ndim=2)
+            for i, array in items_dict["CategoricalBlock"]
         ]
         blocks.extend(cat_blocks)
 
     if len(items_dict["ExtensionBlock"]):
         external_blocks = [
-            make_block(array, klass=ExtensionBlock, placement=i, ndim=2)
-            for i, _, array in items_dict["ExtensionBlock"]
+            make_block(array, klass=ExtensionBlock, placement=slice(*i), ndim=2)
+            for i, array in items_dict["ExtensionBlock"]
         ]
 
         blocks.extend(external_blocks)
 
     if len(items_dict["ObjectValuesExtensionBlock"]):
         external_blocks = [
-            make_block(array, klass=ObjectValuesExtensionBlock, placement=i, ndim=2)
-            for i, _, array in items_dict["ObjectValuesExtensionBlock"]
+            make_block(
+                array, klass=ObjectValuesExtensionBlock, placement=slice(*i), ndim=2
+            )
+            for i, array in items_dict["ObjectValuesExtensionBlock"]
         ]
 
         blocks.extend(external_blocks)
@@ -1849,10 +1839,10 @@ def _simple_blockify(tuples, dtype) -> List[Block]:
     return [block]
 
 
-def _multi_blockify(tuples, dtype: Optional[Dtype] = None):
+def _multi_blockify(tuples):
     """ return an array of blocks that potentially have different dtypes """
     # group by dtype
-    grouper = itertools.groupby(tuples, lambda x: x[2].dtype)
+    grouper = itertools.groupby(tuples, lambda x: x[1].dtype)
 
     new_blocks = []
     for dtype, tup_block in grouper:
@@ -1880,16 +1870,27 @@ def _shape_compat(x) -> Shape:
         else:
             return x.shape
 
-    placement, names, arrays = zip(*tuples)
+    placement, arrays = zip(*tuples)
 
     first = arrays[0]
-    shape = (len(arrays),) + _shape_compat(first)
 
+    # GH#39263
+    # if only one array is passed, avoid copying it
+    if len(arrays) == 1:
+        arr = _asarray_compat(first)
+        # except if the dtype doesn't match
+        if dtype is not None and not is_dtype_equal(first.dtype, dtype):
+            arr = _asarray_compat(first).astype(dtype)
+        if len(_shape_compat(first)) < 2:
+            arr = arr.reshape(1, -1)
+        return arr, slice(*placement[0])
+
+    shape = (len(arrays),) + _shape_compat(first)
     stacked = np.empty(shape, dtype=dtype)
     for i, arr in enumerate(arrays):
         stacked[i] = _asarray_compat(arr)
 
-    return stacked, placement
+    return stacked, list(itertools.chain(*(list(range(*p)) for p in placement)))
 
 
 def _interleaved_dtype(blocks: Sequence[Block]) -> Optional[DtypeObj]:

pandas/tests/frame/test_constructors.py

@@ -1837,10 +1837,20 @@ def test_constructor_ndarray_copy(self, float_frame):
         float_frame.values[6] = 6
         assert not (df.values[6] == 6).all()
 
-    def test_constructor_series_copy(self, float_frame):
+    def test_constructor_series_not_copy(self, float_frame):
         series = float_frame._series
 
         df = DataFrame({"A": series["A"]})
+
+        df["A"][:] = 5
+
+        assert (series["A"] == 5).all()
+
+    def test_constructor_series_copy(self, float_frame):
+        series = float_frame._series
+
+        df = DataFrame({"A": series["A"], "B": series["A"]})
+
         df["A"][:] = 5
 
         assert not (series["A"] == 5).all()