BUG: DataFrame.groupby with as_index=False shouldn't modify grouping columns

Author: rhshadrach

doc/source/whatsnew/v1.1.0.rst

@@ -803,6 +803,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.groupby` where a ``ValueError`` would be raised when grouping by a categorical column with read-only categories and ``sort=False`` (:issue:`33410`)
 - Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
 - Bug in :meth:`Rolling.min` and :meth:`Rolling.max`: Growing memory usage after multiple calls when using a fixed window (:issue:`30726`)
+- Bug in :meth:`DataFrame.groupby` when using ``as_index=False`` would modify the grouping column when used with ``idxmax``, ``idxmin``, ``mad``, ``nunique``, and ``skew`` (:issue:`21090`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/groupby/generic.py

@@ -1257,7 +1257,7 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
 
             v = values[0]
 
-            if isinstance(v, (np.ndarray, Index, Series)):
+            if isinstance(v, (np.ndarray, Index, Series)) or not self.as_index:
                 if isinstance(v, Series):
                     applied_index = self._selected_obj._get_axis(self.axis)
                     all_indexed_same = all_indexes_same([x.index for x in values])
@@ -1333,6 +1333,11 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
                     result = DataFrame(
                         stacked_values.T, index=v.index, columns=key_index
                     )
+                elif not self.as_index:
+                    # We add grouping column below, so create a frame here
+                    result = DataFrame(
+                        values, index=key_index, columns=[self._selection]
+                    )
                 else:
                     # GH#1738: values is list of arrays of unequal lengths
                     #  fall through to the outer else clause
@@ -1348,6 +1353,9 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
                 else:
                     result = result._convert(datetime=True)
 
+                if not self.as_index:
+                    self._insert_inaxis_grouper_inplace(result)
+
                 return self._reindex_output(result)
 
             # values are not series or array-like but scalars
@@ -1684,9 +1692,11 @@ def _insert_inaxis_grouper_inplace(self, result):
                 ),
             )
         )
-
+        columns = result.columns
         for name, lev, in_axis in izip:
-            if in_axis:
+            # GH #28549
+            # When using .apply(-), name will be in columns already
+            if in_axis and name not in columns:
                 result.insert(0, name, lev)
 
     def _wrap_aggregated_output(
@@ -1836,11 +1846,11 @@ def nunique(self, dropna: bool = True):
         5   ham       5      y
 
         >>> df.groupby('id').nunique()
-            id  value1  value2
+              value1  value2
         id
-        egg    1       1       1
-        ham    1       1       2
-        spam   1       2       1
+        egg        1       1
+        ham        1       2
+        spam       2       1
 
         Check for rows with the same id but conflicting values:
 
@@ -1851,7 +1861,7 @@ def nunique(self, dropna: bool = True):
         4   ham       5      x
         5   ham       5      y
         """
-        obj = self._selected_obj
+        obj = self._obj_with_exclusions
 
         def groupby_series(obj, col=None):
             return SeriesGroupBy(obj, selection=col, grouper=self.grouper).nunique(
@@ -1882,6 +1892,9 @@ def groupby_series(obj, col=None):
 
         if not self.as_index:
             results.index = ibase.default_index(len(results))
+            if results.ndim == 1:
+                results = results.to_frame()
+            self._insert_inaxis_grouper_inplace(results)
         return results
 
     boxplot = boxplot_frame_groupby

pandas/core/groupby/groupby.py

@@ -718,11 +718,11 @@ def _make_wrapper(self, name):
 
         # need to setup the selection
         # as are not passed directly but in the grouper
-        f = getattr(self._selected_obj, name)
+        f = getattr(self._obj_with_exclusions, name)
         if not isinstance(f, types.MethodType):
             return self.apply(lambda self: getattr(self, name))
 
-        f = getattr(type(self._selected_obj), name)
+        f = getattr(type(self._obj_with_exclusions), name)
         sig = inspect.signature(f)
 
         def wrapper(*args, **kwargs):
@@ -745,7 +745,7 @@ def curried(x):
                 return self.apply(curried)
 
             try:
-                return self.apply(curried)
+                return self._python_apply_general(curried, self._obj_with_exclusions)
             except TypeError as err:
                 if not re.search(
                     "reduction operation '.*' not allowed for this dtype", str(err)
@@ -836,7 +836,7 @@ def f(g):
         # ignore SettingWithCopy here in case the user mutates
         with option_context("mode.chained_assignment", None):
             try:
-                result = self._python_apply_general(f)
+                result = self._python_apply_general(f, self._selected_obj)
             except TypeError:
                 # gh-20949
                 # try again, with .apply acting as a filtering
@@ -847,12 +847,12 @@ def f(g):
                 # on a string grouper column
 
                 with _group_selection_context(self):
-                    return self._python_apply_general(f)
+                    return self._python_apply_general(f, self._selected_obj)
 
         return result
 
-    def _python_apply_general(self, f):
-        keys, values, mutated = self.grouper.apply(f, self._selected_obj, self.axis)
+    def _python_apply_general(self, f, obj):
+        keys, values, mutated = self.grouper.apply(f, obj, self.axis)
 
         return self._wrap_applied_output(
             keys, values, not_indexed_same=mutated or self.mutated
@@ -1019,7 +1019,7 @@ def _python_agg_general(
             output[key] = maybe_cast_result(result, obj, numeric_only=True)
 
         if len(output) == 0:
-            return self._python_apply_general(f)
+            return self._python_apply_general(f, self._selected_obj)
 
         if self.grouper._filter_empty_groups:
 

pandas/tests/groupby/test_function.py

@@ -280,7 +280,7 @@ def test_non_cython_api():
     result = g.mad()
     tm.assert_frame_equal(result, expected)
 
-    expected = DataFrame([[0.0, 0.0], [0, np.nan]], columns=["A", "B"], index=[0, 1])
+    expected = DataFrame([[1, 0.0], [3, np.nan]], columns=["A", "B"], index=[0, 1])
     result = gni.mad()
     tm.assert_frame_equal(result, expected)
 
@@ -573,6 +573,32 @@ def test_ops_general(op, targop):
     tm.assert_frame_equal(result, expected)
 
 
+def test_ops_not_as_index(reduction_func):
+    # GH 21090
+    # Using as_index=False should not modify grouped column
+
+    if reduction_func in ("corrwith",):
+        pytest.skip("Test not applicable")
+
+    if reduction_func in ("nth", "ngroup", "size",):
+        pytest.skip("Skip until behavior is determined (GH #5755)")
+
+    if reduction_func in ("sem", "std"):
+        pytest.skip("Function incorrectly modifies keys (GH #10355)")
+
+    df = DataFrame(np.random.randint(0, 5, size=(100, 2)), columns=["a", "b"])
+    expected = getattr(df.groupby("a"), reduction_func)().reset_index()
+
+    result = getattr(df.groupby("a", as_index=False), reduction_func)()
+    tm.assert_frame_equal(result, expected)
+
+    result = df.groupby("a", as_index=False).agg(reduction_func)
+    tm.assert_frame_equal(result, expected)
+
+    result = getattr(df.groupby("a", as_index=False)["b"], reduction_func)()
+    tm.assert_frame_equal(result, expected)
+
+
 def test_max_nan_bug():
     raw = """,Date,app,File
 -04-23,2013-04-23 00:00:00,,log080001.log

pandas/tests/groupby/test_whitelist.py

@@ -406,7 +406,7 @@ def test_all_methods_categorized(mframe):
     if new_names:
         msg = f"""
 There are uncatgeorized methods defined on the Grouper class:
-{names}.
+{new_names}.
 
 Was a new method recently added?