Fixed groupby().apply(func) bug when working with time colums (GH #11324)

Addressed PR comments

Added comments and updated whatsnew

Author: robdmc

doc/source/whatsnew/v0.17.1.txt

@@ -117,3 +117,5 @@ Bug Fixes
 - Bug in ``to_excel`` with openpyxl 2.2+ and merging (:issue:`11408`)
 
 - Bug in ``DataFrame.to_dict()`` produces a ``np.datetime64`` object instead of ``Timestamp`` when only datetime is present in data (:issue:`11327`)
+
+- Bug in ``pandas.core.groupby`` raises exception when ``func`` in ``df.groupby(...).apply(func)`` doesn't return existing time columns (:issue:`11324`)

pandas/core/groupby.py

@@ -3124,6 +3124,7 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
                     result = result._convert(numeric=True)
                     date_cols = self._selected_obj.select_dtypes(
                         include=list(_DATELIKE_DTYPES)).columns
+                    date_cols = date_cols.intersection(result.columns)
                     result[date_cols] = (result[date_cols]
                                          ._convert(datetime=True,
                                                           coerce=True))

pandas/tests/test_groupby.py

@@ -824,6 +824,30 @@ def test_apply_issues(self):
         result = df.groupby('date').apply(lambda x: x['time'][x['value'].idxmax()])
         assert_series_equal(result, expected)
 
+    def test_time_field_bug(self):
+        # Test a fix for the following error related to GH issue 11324
+        # When non-key fields in a group-by dataframe contained time-based fields that
+        # were not returned by the apply function, an exception would be raised.
+
+        df = pd.DataFrame({'a': 1,'b': [datetime.now() for nn in range(10)]})
+
+        def func_with_no_date(batch):
+            return pd.Series({'c': 2})
+
+        def func_with_date(batch):
+            return pd.Series({'c': 2, 'b': datetime(2015, 1, 1)})
+
+        dfg_no_conversion = df.groupby(by=['a']).apply(func_with_no_date)
+        dfg_no_conversion_expected = pd.DataFrame({'c': 2}, index=[1])
+        dfg_no_conversion_expected.index.name = 'a'
+
+        dfg_conversion = df.groupby(by=['a']).apply(func_with_date)
+        dfg_conversion_expected = pd.DataFrame({'b': datetime(2015, 1, 1), 'c': 2}, index=[1])
+        dfg_conversion_expected.index.name = 'a'
+
+        self.assert_frame_equal(dfg_no_conversion, dfg_no_conversion_expected)
+        self.assert_frame_equal(dfg_conversion, dfg_conversion_expected)
+
     def test_len(self):
         df = tm.makeTimeDataFrame()
         grouped = df.groupby([lambda x: x.year,