BUG: locset with Series as column key fails inconsistently

[sfc-gh-rdurrani]

Pandas version checks

• I have checked that this issue has not already been reported.

• I have confirmed this bug exists on the latest version of pandas.

• I have confirmed this bug exists on the main branch of pandas.

Reproducible Example

>>> import pandas as pd

>>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=list("ABC"))

>>>  df.loc[:, pd.Series(['A', 'C'])] = pd.Series([10, 20, 30]) # succeeds

>>> df
    A  B   C
0  10  2  20
1  10  5  20

>>> df.loc[:, pd.Series(['A', 'B', 'C'])] = pd.Series([10, 20, 30]) # fails
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/blocks.py:1429, in Block.setitem(self, indexer, value, using_cow)
   1428 try:
-> 1429     values[indexer] = casted
   1430 except (TypeError, ValueError) as err:

ValueError: shape mismatch: value array of shape (2,) could not be broadcast to indexing result of shape (2,3)

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
Cell In[5], line 1
----> 1 df.loc[:, pd.Series(['A', 'B',  'C'])] = pd.Series([10, 20, 30])

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/indexing.py:911, in _LocationIndexer.__setitem__(self, key, value)
    908 self._has_valid_setitem_indexer(key)
    910 iloc = self if self.name == "iloc" else self.obj.iloc
--> 911 iloc._setitem_with_indexer(indexer, value, self.name)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/indexing.py:1944, in _iLocIndexer._setitem_with_indexer(self, indexer, value, name)
   1942     self._setitem_with_indexer_split_path(indexer, value, name)
   1943 else:
-> 1944     self._setitem_single_block(indexer, value, name)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/indexing.py:2218, in _iLocIndexer._setitem_single_block(self, indexer, value, name)
   2215 self.obj._check_is_chained_assignment_possible()
   2217 # actually do the set
-> 2218 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
   2219 self.obj._maybe_update_cacher(clear=True, inplace=True)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/managers.py:415, in BaseBlockManager.setitem(self, indexer, value, warn)
    411     # No need to split if we either set all columns or on a single block
    412     # manager
    413     self = self.copy()
--> 415 return self.apply("setitem", indexer=indexer, value=value)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/managers.py:363, in BaseBlockManager.apply(self, f, align_keys, **kwargs)
    361         applied = b.apply(f, **kwargs)
    362     else:
--> 363         applied = getattr(b, f)(**kwargs)
    364     result_blocks = extend_blocks(applied, result_blocks)
    366 out = type(self).from_blocks(result_blocks, self.axes)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/blocks.py:1432, in Block.setitem(self, indexer, value, using_cow)
   1430     except (TypeError, ValueError) as err:
   1431         if is_list_like(casted):
-> 1432             raise ValueError(
   1433                 "setting an array element with a sequence."
   1434             ) from err
   1435         raise
   1436 return self

ValueError: setting an array element with a sequence.

>>> In [7]: df.loc[:, pd.Series(['A'])] = pd.Series([10, 20, 30]) # Fails
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/blocks.py:1429, in Block.setitem(self, indexer, value, using_cow)
   1428 try:
-> 1429     values[indexer] = casted
   1430 except (TypeError, ValueError) as err:

ValueError: shape mismatch: value array of shape (2,) could not be broadcast to indexing result of shape (2,1)

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
Cell In[7], line 1
----> 1 df.loc[:, pd.Series(['A'])] = pd.Series([10, 20, 30])

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/indexing.py:911, in _LocationIndexer.__setitem__(self, key, value)
    908 self._has_valid_setitem_indexer(key)
    910 iloc = self if self.name == "iloc" else self.obj.iloc
--> 911 iloc._setitem_with_indexer(indexer, value, self.name)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/indexing.py:1944, in _iLocIndexer._setitem_with_indexer(self, indexer, value, name)
   1942     self._setitem_with_indexer_split_path(indexer, value, name)
   1943 else:
-> 1944     self._setitem_single_block(indexer, value, name)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/indexing.py:2218, in _iLocIndexer._setitem_single_block(self, indexer, value, name)
   2215 self.obj._check_is_chained_assignment_possible()
   2217 # actually do the set
-> 2218 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
   2219 self.obj._maybe_update_cacher(clear=True, inplace=True)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/managers.py:415, in BaseBlockManager.setitem(self, indexer, value, warn)
    411     # No need to split if we either set all columns or on a single block
    412     # manager
    413     self = self.copy()
--> 415 return self.apply("setitem", indexer=indexer, value=value)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/managers.py:363, in BaseBlockManager.apply(self, f, align_keys, **kwargs)
    361         applied = b.apply(f, **kwargs)
    362     else:
--> 363         applied = getattr(b, f)(**kwargs)
    364     result_blocks = extend_blocks(applied, result_blocks)
    366 out = type(self).from_blocks(result_blocks, self.axes)

File ~/.miniconda3/envs/snowpark/lib/python3.10/site-packages/pandas/core/internals/blocks.py:1432, in Block.setitem(self, indexer, value, using_cow)
   1430     except (TypeError, ValueError) as err:
   1431         if is_list_like(casted):
-> 1432             raise ValueError(
   1433                 "setting an array element with a sequence."
   1434             ) from err
   1435         raise
   1436 return self

ValueError: setting an array element with a sequence

>>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=list("ABC"))

>>> df.loc[:, pd.Series(['A', 'B', 'C'])] = pd.Series([10, 20, 30]) # Succeeds

>>> df
    A   B   C
0  10  20  30
1  10  20  30
2  10  20  30

Issue Description

It seems that we can only provide as many column indexers in the Series key for the column as are rows, but all of the above examples should succeed.

Expected Behavior

Columns that are indexed should be set with the corresponding value (determined by position).

Installed Versions

INSTALLED VERSIONS

commit : bdc79c1
python : 3.10.14.final.0
python-bits : 64
OS : Darwin
OS-release : 23.6.0
Version : Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:04 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6020
machine : arm64
processor : arm
byteorder : little
LC_ALL : None
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8

pandas : 2.2.1
numpy : 1.26.4
pytz : 2024.1
dateutil : 2.9.0.post0
setuptools : 72.1.0
pip : 24.2
Cython : None
pytest : 7.4.4
hypothesis : None
sphinx : 5.0.2
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 3.1.4
IPython : 8.27.0
pandas_datareader : None
adbc-driver-postgresql: None
adbc-driver-sqlite : None
bs4 : 4.12.3
bottleneck : None
dataframe-api-compat : None
fastparquet : None
fsspec : 2024.6.1
gcsfs : None
matplotlib : 3.9.2
numba : None
numexpr : None
odfpy : None
openpyxl : 3.1.5
pandas_gbq : None
pyarrow : 17.0.0
pyreadstat : None
python-calamine : None
pyxlsb : None
s3fs : None
scipy : 1.14.1
sqlalchemy : None
tables : None
tabulate : None
xarray : None
xlrd : None
zstandard : None
tzdata : 2024.1
qtpy : None
pyqt5 : None

[rhshadrach]

Thanks for the report. I'm not sure this was ever meant to be supported. From the documentation: https://pandas.pydata.org/docs/dev/reference/api/pandas.DataFrame.loc.html

The (relevant) supported arguments are:

• A list or array of labels
• An alignable boolean Series. The index of the key will be aligned before masking.

While you are providing a Series, it is not a Boolean Series. However

df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=list("ABC"))
df.loc[:, pd.Series({"A": True, "B": True, "C": True})] = pd.Series([10, 20, 30])

has the same issue. It appears that the preparation in core.indexing is done to set the values [10 20] column-wise (so that [10 20] will be in each column), but then the assignment happens row-wise.

Further investigations and PRs to fix are welcome!

[vamsi-verma-s]

I tried to investigate a bit more when i saw df.loc[:, pd.Series(['A', 'B', 'C'])] = pd.Series([10, 20, 30]) # fails . This looks like it should work and does work when using .iloc.

df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=list("ABC"))
df.iloc[:, pd.Series([0, 1, 2])] = pd.Series([10, 20, 30])

    A   B   C
0  10  20  30
1  10  20  30

Therefore, you might want to use .iloc which seems to have more consistent behavior.

---

where this is stemming from - i see a difference in behavior between .loc and .iloc. where they deviate is, _align_series method which is only called for .loc where the expectation is that the Series will be indexed according to the indexer provided. But, if only a column indexer slice is provided, it is not considered (see below) and the Series is indexed using the row Slice, which brings the length of rows into the picture.

pandas/pandas/core/indexing.py

Lines 2173 to 2179 in 05fa958

	if (isinstance(value, ABCSeries) and name != "iloc") or isinstance(value, dict):
	# TODO(EA): ExtensionBlock.setitem this causes issues with
	# setting for extensionarrays that store dicts. Need to decide
	# if it's worth supporting that.
	value = self._align_series(indexer, Series(value))

pandas/pandas/core/indexing.py

Lines 2357 to 2371 in 05fa958

	aligners = [not com.is_null_slice(idx) for idx in indexer]
	sum_aligners = sum(aligners)
	single_aligner = sum_aligners == 1
	is_frame = self.ndim == 2
	obj = self.obj
	# are we a single alignable value on a non-primary
	# dim (e.g. panel: 1,2, or frame: 0) ?
	# hence need to align to a single axis dimension
	# rather that find all valid dims
	# frame
	if is_frame:
	single_aligner = single_aligner and aligners[0]

Even if you try to provide both the slices instead of just column slice to .loc which causes the issue, it does not work as expected.

In [35]: df.loc[df.index, pd.Series(['A', 'B', 'C'])] = pd.Series([10, 20, 30])

In [36]: df
Out[36]: 
    A   B   C
0  10  10  10
1  20  20  20

._align_series method creates the above dataframe value from the Series which does not seem correct.

[anzber]

take

[anzber]

@rhshadrach

I've found that assignment behaves differently depending on indexing. However, it is not clear how exactly this should work.

Case 1. Indexing with boolean Series

>>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=list("ABC"))
>>> df.loc[df['A']>0, ['A', 'B', 'C']] = pd.Series([10, 20, 30]) 
>>> df
    A   B   C
0  10  10  10
1  20  20  20
2  30  30  30

Case 2. Indexing with slice (should be equivalent with boolean array, but not)

>>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=list("ABC"))
>>> df.loc[:, ['A', 'B', 'C']] = pd.Series([10, 20, 30]) 
>>> df
    A   B   C
0  10  20  30
1  10  20  30
2  10  20  30

I propose to define it this way: for cases where we assign a vector (Series or list or smth else) to an array and the length of the vector is equal to the number of target columns the result should be similar to Case 2

[anzber]

Found a test that confirms the correctness of Case 1.

pandas/pandas/tests/indexing/test_indexing.py

Lines 475 to 492 in 05fa958

	def test_multi_assign_broadcasting_rhs(self):
	# broadcasting on the rhs is required
	df = DataFrame(
	{
	"A": [1, 2, 0, 0, 0],
	"B": [0, 0, 0, 10, 11],
	"C": [0, 0, 0, 10, 11],
	"D": [3, 4, 5, 6, 7],
	}
	)
	expected = df.copy()
	mask = expected["A"] == 0
	for col in ["A", "B"]:
	expected.loc[mask, col] = df["D"]
	df.loc[df["A"] == 0, ["A", "B"]] = df["D"].copy()
	tm.assert_frame_equal(df, expected)

[anzber]

Comments about pull-request:

1. Fixed case 2 above with inconsistent behavior of different indexes

>>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=list("ABC"))
>>> df.loc[:, ['A', 'B', 'C']] = pd.Series([10, 20, 30]) 
>>> df
    A   B   C
0  10  10  10
1  20  20  20
2  30  30  30

2. Fixed initial issue, but it works consistent with test_multi_assign_broadcasting_rhs like this

>>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=list("ABC"))
>>> df.loc[:, pd.Series(['A', 'B', 'C'])] = pd.Series([10, 20, 30])
>>> df
    A   B   C
0  10  10  10
1  20  20  20