diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py
index 90fb3b8685c93..141aac51a624f 100644
--- a/pandas/core/ops/__init__.py
+++ b/pandas/core/ops/__init__.py
@@ -787,11 +787,17 @@ def na_op(x, y):
 
         return result
 
-    fill_int = lambda x: x.fillna(0)
+    fill_int = lambda x: x
 
     def fill_bool(x, left=None):
         # if `left` is specifically not-boolean, we do not cast to bool
-        x = x.fillna(False)
+        if x.dtype.kind in ["c", "f", "O"]:
+            # dtypes that can hold NA
+            mask = isna(x)
+            if mask.any():
+                x = x.astype(object)
+                x[mask] = False
+
         if left is None or is_bool_dtype(left.dtype):
             x = x.astype(bool)
         return x
@@ -814,40 +820,35 @@ def wrapper(self, other):
             # Defer to DataFrame implementation; fail early
             return NotImplemented
 
-        elif should_extension_dispatch(self, other):
-            lvalues = extract_array(self, extract_numpy=True)
-            rvalues = extract_array(other, extract_numpy=True)
+        other = lib.item_from_zerodim(other)
+        if is_list_like(other) and not hasattr(other, "dtype"):
+            # e.g. list, tuple
+            other = construct_1d_object_array_from_listlike(other)
+
+        lvalues = extract_array(self, extract_numpy=True)
+        rvalues = extract_array(other, extract_numpy=True)
+
+        if should_extension_dispatch(self, rvalues):
             res_values = dispatch_to_extension_op(op, lvalues, rvalues)
-            result = self._constructor(res_values, index=self.index, name=res_name)
-            return finalizer(result)
 
-        elif isinstance(other, (ABCSeries, ABCIndexClass)):
-            is_other_int_dtype = is_integer_dtype(other.dtype)
-            other = other if is_other_int_dtype else fill_bool(other, self)
+        else:
+            if isinstance(rvalues, (ABCSeries, ABCIndexClass, np.ndarray)):
+                is_other_int_dtype = is_integer_dtype(rvalues.dtype)
+                rvalues = rvalues if is_other_int_dtype else fill_bool(rvalues, lvalues)
 
-        elif is_list_like(other):
-            # list, tuple, np.ndarray
-            if not isinstance(other, np.ndarray):
-                other = construct_1d_object_array_from_listlike(other)
+            else:
+                # i.e. scalar
+                is_other_int_dtype = lib.is_integer(rvalues)
 
-            is_other_int_dtype = is_integer_dtype(other.dtype)
-            other = type(self)(other)
-            other = other if is_other_int_dtype else fill_bool(other, self)
+            # For int vs int `^`, `|`, `&` are bitwise operators and return
+            #   integer dtypes.  Otherwise these are boolean ops
+            filler = fill_int if is_self_int_dtype and is_other_int_dtype else fill_bool
 
-        else:
-            # i.e. scalar
-            is_other_int_dtype = lib.is_integer(other)
-
-        # TODO: use extract_array once we handle EA correctly, see GH#27959
-        ovalues = lib.values_from_object(other)
-
-        # For int vs int `^`, `|`, `&` are bitwise operators and return
-        #   integer dtypes.  Otherwise these are boolean ops
-        filler = fill_int if is_self_int_dtype and is_other_int_dtype else fill_bool
-        res_values = na_op(self.values, ovalues)
-        unfilled = self._constructor(res_values, index=self.index, name=res_name)
-        filled = filler(unfilled)
-        return finalizer(filled)
+            res_values = na_op(lvalues, rvalues)
+            res_values = filler(res_values)
+
+        result = self._constructor(res_values, index=self.index, name=res_name)
+        return finalizer(result)
 
     wrapper.__name__ = op_name
     return wrapper