API: Allow equality comparisons of Series with a categorical dtype and object dtype are allowed (previously would raise TypeError) (GH8938)

Author: jreback

doc/source/whatsnew/v0.15.2.txt

@@ -59,6 +59,8 @@ API changes
      p = pd.Panel(np.random.rand(2, 5, 4) > 0.1)
      p.all()
 
+- Allow equality comparisons of Series with a categorical dtype and object dtype; previously these would raise ``TypeError`` (:issue:`8938`)
+
 .. _whatsnew_0152.enhancements:
 
 Enhancements

pandas/core/categorical.py

@@ -64,6 +64,12 @@ def f(self, other):
             else:
                 return np.repeat(False, len(self))
         else:
+
+            # allow categorical vs object dtype array comparisons for equality
+            # these are only positional comparisons
+            if op in ['__eq__','__ne__']:
+                return getattr(np.array(self),op)(np.array(other))
+
             msg = "Cannot compare a Categorical for op {op} with type {typ}. If you want to \n" \
                   "compare values, use 'np.asarray(cat) <op> other'."
             raise TypeError(msg.format(op=op,typ=type(other)))

pandas/core/ops.py

@@ -541,10 +541,13 @@ def _comp_method_SERIES(op, name, str_rep, masker=False):
     """
     def na_op(x, y):
 
-        if com.is_categorical_dtype(x) != (not np.isscalar(y) and com.is_categorical_dtype(y)):
-            msg = "Cannot compare a Categorical for op {op} with type {typ}. If you want to \n" \
-                  "compare values, use 'series <op> np.asarray(cat)'."
-            raise TypeError(msg.format(op=op,typ=type(y)))
+        # dispatch to the categorical if we have a categorical
+        # in either operand
+        if com.is_categorical_dtype(x):
+            return op(x,y)
+        elif com.is_categorical_dtype(y) and not lib.isscalar(y):
+            return op(y,x)
+
         if x.dtype == np.object_:
             if isinstance(y, list):
                 y = lib.list_to_object_array(y)
@@ -586,33 +589,33 @@ def wrapper(self, other):
                 msg = "Cannot compare a Categorical for op {op} with Series of dtype {typ}.\n"\
                       "If you want to compare values, use 'series <op> np.asarray(other)'."
                 raise TypeError(msg.format(op=op,typ=self.dtype))
-        else:
 
-            mask = isnull(self)
 
-            values = self.get_values()
-            other = _index.convert_scalar(values,_values_from_object(other))
+        mask = isnull(self)
 
-            if issubclass(values.dtype.type, (np.datetime64, np.timedelta64)):
-                values = values.view('i8')
+        values = self.get_values()
+        other = _index.convert_scalar(values,_values_from_object(other))
 
-            # scalars
-            res = na_op(values, other)
-            if np.isscalar(res):
-                raise TypeError('Could not compare %s type with Series'
-                                % type(other))
+        if issubclass(values.dtype.type, (np.datetime64, np.timedelta64)):
+            values = values.view('i8')
 
-            # always return a full value series here
-            res = _values_from_object(res)
+        # scalars
+        res = na_op(values, other)
+        if np.isscalar(res):
+            raise TypeError('Could not compare %s type with Series'
+                            % type(other))
 
-            res = pd.Series(res, index=self.index, name=self.name,
-                            dtype='bool')
+        # always return a full value series here
+        res = _values_from_object(res)
 
-            # mask out the invalids
-            if mask.any():
-                res[mask] = masker
+        res = pd.Series(res, index=self.index, name=self.name,
+                        dtype='bool')
+
+        # mask out the invalids
+        if mask.any():
+            res[mask] = masker
 
-            return res
+        return res
     return wrapper
 
 

pandas/tests/test_categorical.py

@@ -2211,11 +2211,63 @@ def f():
             tm.assert_series_equal(res, exp)
 
         # And test NaN handling...
-        cat = pd.Series(pd.Categorical(["a","b","c", np.nan]))
+        cat = Series(Categorical(["a","b","c", np.nan]))
         exp = Series([True, True, True, False])
         res = (cat == cat)
         tm.assert_series_equal(res, exp)
 
+    def test_cat_equality(self):
+
+        # GH 8938
+        # allow equality comparisons
+        a = Series(list('abc'),dtype="category")
+        b = Series(list('abc'),dtype="object")
+        c = Series(['a','b','cc'],dtype="object")
+        d = Series(list('acb'),dtype="object")
+        e = Categorical(list('abc'))
+        f = Categorical(list('acb'))
+
+        # vs scalar
+        self.assertFalse((a=='a').all())
+        self.assertTrue(((a!='a') == ~(a=='a')).all())
+
+        self.assertFalse(('a'==a).all())
+        self.assertTrue((a=='a')[0])
+        self.assertTrue(('a'==a)[0])
+        self.assertFalse(('a'!=a)[0])
+
+        # vs list-like
+        self.assertTrue((a==a).all())
+        self.assertFalse((a!=a).all())
+
+        self.assertTrue((a==list(a)).all())
+        self.assertTrue((a==b).all())
+        self.assertTrue((b==a).all())
+        self.assertTrue(((~(a==b))==(a!=b)).all())
+        self.assertTrue(((~(b==a))==(b!=a)).all())
+
+        self.assertFalse((a==c).all())
+        self.assertFalse((c==a).all())
+        self.assertFalse((a==d).all())
+        self.assertFalse((d==a).all())
+
+        # vs a cat-like
+        self.assertTrue((a==e).all())
+        self.assertTrue((e==a).all())
+        self.assertFalse((a==f).all())
+        self.assertFalse((f==a).all())
+
+        self.assertTrue(((~(a==e)==(a!=e)).all()))
+        self.assertTrue(((~(e==a)==(e!=a)).all()))
+        self.assertTrue(((~(a==f)==(a!=f)).all()))
+        self.assertTrue(((~(f==a)==(f!=a)).all()))
+
+        # non-equality is not comparable
+        self.assertRaises(TypeError, lambda: a < b)
+        self.assertRaises(TypeError, lambda: b < a)
+        self.assertRaises(TypeError, lambda: a > b)
+        self.assertRaises(TypeError, lambda: b > a)
+
     def test_concat(self):
         cat = pd.Categorical(["a","b"], categories=["a","b"])
         vals = [1,2]