TST: fix up pivot tests

Author: jreback

doc/source/whatsnew/v0.23.0.txt

@@ -531,6 +531,20 @@ New Behavior:
 
    df.groupby(['A', 'B', 'C']).count()
 
+Furthermore, the result of pivotting will now only show observed values. ``dropna`` will have no
+effect for a Categorical pivotted column.
+
+.. ipython:: python
+
+   categories = ['a', 'b', 'c', 'd']
+   from pandas.api.types import CategoricalDtype as CDT
+   df = DataFrame({'A': ['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c'],
+                   'B': [1, 2, 3, 1, 2, 3, 1, 2, 3],
+                   'C': range(0, 9)})
+
+   df['A'] = df['A'].astype(CDT(categories, ordered=False))
+   df.pivot_table(index='B', columns='A', values='C')
+
 .. _whatsnew_0230.api_breaking.deprecate_panel:
 
 Deprecate Panel

pandas/tests/reshape/test_pivot.py

@@ -93,23 +93,23 @@ def test_pivot_table_dropna(self):
 
     def test_pivot_table_categorical(self):
 
-        raw_cat1 = Categorical(["a", "a", "b", "b"],
-                               categories=["a", "b", "z"], ordered=True)
-        raw_cat2 = Categorical(["c", "d", "c", "d"],
-                               categories=["c", "d", "y"], ordered=True)
-        df = DataFrame({"A": raw_cat1, "B": raw_cat2, "values": [1, 2, 3, 4]})
+        cat1 = Categorical(["a", "a", "b", "b"],
+                           categories=["a", "b", "z"], ordered=True)
+        cat2 = Categorical(["c", "d", "c", "d"],
+                           categories=["c", "d", "y"], ordered=True)
+        df = DataFrame({"A": cat1, "B": cat2, "values": [1, 2, 3, 4]})
         result = pd.pivot_table(df, values='values', index=['A', 'B'])
 
-        exp_index = pd.MultiIndex.from_product(
-            [Categorical(["a", "b", "z"], ordered=True),
-             Categorical(["c", "d", "y"], ordered=True)],
+        exp_index = pd.MultiIndex.from_arrays(
+            [cat1, cat2],
             names=['A', 'B'])
         expected = DataFrame(
-            {'values': [1, 2, np.nan, 3, 4, np.nan, np.nan, np.nan, np.nan]},
+            {'values': [1, 2, 3, 4]},
             index=exp_index)
         tm.assert_frame_equal(result, expected)
 
-    def test_pivot_table_dropna_categoricals(self):
+    @pytest.mark.parametrize('dropna', [True, False])
+    def test_pivot_table_dropna_categoricals(self, dropna):
         # GH 15193
         categories = ['a', 'b', 'c', 'd']
 
@@ -118,30 +118,18 @@ def test_pivot_table_dropna_categoricals(self):
                         'C': range(0, 9)})
 
         df['A'] = df['A'].astype(CDT(categories, ordered=False))
-        result_true = df.pivot_table(index='B', columns='A', values='C',
-                                     dropna=True)
+        result = df.pivot_table(index='B', columns='A', values='C',
+                                dropna=dropna)
         expected_columns = Series(['a', 'b', 'c'], name='A')
         expected_columns = expected_columns.astype(
             CDT(categories, ordered=False))
         expected_index = Series([1, 2, 3], name='B')
-        expected_true = DataFrame([[0.0, 3.0, 6.0],
-                                   [1.0, 4.0, 7.0],
-                                   [2.0, 5.0, 8.0]],
-                                  index=expected_index,
-                                  columns=expected_columns,)
-        tm.assert_frame_equal(expected_true, result_true)
-
-        result_false = df.pivot_table(index='B', columns='A', values='C',
-                                      dropna=False)
-        expected_columns = (
-            Series(['a', 'b', 'c', 'd'], name='A').astype('category')
-        )
-        expected_false = DataFrame([[0.0, 3.0, 6.0, np.NaN],
-                                    [1.0, 4.0, 7.0, np.NaN],
-                                    [2.0, 5.0, 8.0, np.NaN]],
-                                   index=expected_index,
-                                   columns=expected_columns,)
-        tm.assert_frame_equal(expected_false, result_false)
+        expected = DataFrame([[0, 3, 6],
+                              [1, 4, 7],
+                              [2, 5, 8]],
+                             index=expected_index,
+                             columns=expected_columns,)
+        tm.assert_frame_equal(result, expected)
 
     def test_pass_array(self):
         result = self.data.pivot_table(
@@ -1132,14 +1120,11 @@ def test_categorical_pivot_index_ordering(self):
                                 columns='Year',
                                 aggfunc='sum')
         expected_columns = pd.Int64Index([2013, 2014], name='Year')
-        expected_index = pd.CategoricalIndex(months,
+        expected_index = pd.CategoricalIndex(['January'],
                                              categories=months,
                                              ordered=False,
                                              name='Month')
-        expected_data = np.empty((12, 2))
-        expected_data.fill(np.nan)
-        expected_data[0, :] = [320., 120.]
-        expected = pd.DataFrame(expected_data,
+        expected = pd.DataFrame([[320, 120]],
                                 index=expected_index,
                                 columns=expected_columns)
         tm.assert_frame_equal(result, expected)