diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst
index ef6bb1340dc69..de3941682f539 100644
--- a/doc/source/whatsnew/v2.0.0.rst
+++ b/doc/source/whatsnew/v2.0.0.rst
@@ -1200,6 +1200,7 @@ Indexing
 - Bug in :meth:`Series.loc` raising error for out of bounds end of slice indexer (:issue:`50161`)
 - Bug in :meth:`DataFrame.loc` raising ``ValueError`` with ``bool`` indexer and :class:`MultiIndex` (:issue:`47687`)
 - Bug in :meth:`DataFrame.loc` raising ``IndexError`` when setting values for a pyarrow-backed column with a non-scalar indexer (:issue:`50085`)
+- Bug in :meth:`DataFrame.loc` modifying object when setting incompatible value with an empty indexer (:issue:`45981`)
 - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` when right hand side is :class:`DataFrame` with :class:`MultiIndex` columns (:issue:`49121`)
 - Bug in :meth:`DataFrame.reindex` casting dtype to ``object`` when :class:`DataFrame` has single extension array column when re-indexing ``columns`` and ``index`` (:issue:`48190`)
 - Bug in :meth:`DataFrame.iloc` raising ``IndexError`` when indexer is a :class:`Series` with numeric extension array dtype (:issue:`49521`)
diff --git a/pandas/core/common.py b/pandas/core/common.py
index aaa5134ed1aaa..6713ccd417dd4 100644
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -316,6 +316,18 @@ def is_null_slice(obj) -> bool:
     )
 
 
+def is_empty_slice(obj) -> bool:
+    """
+    We have an empty slice, e.g. no values are selected.
+    """
+    return (
+        isinstance(obj, slice)
+        and obj.start is not None
+        and obj.stop is not None
+        and obj.start == obj.stop
+    )
+
+
 def is_true_slices(line) -> list[bool]:
     """
     Find non-trivial slices in "line": return a list of booleans with same length.
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index 26b5a4077b0ff..8c3e56b686e79 100644
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -2017,7 +2017,13 @@ def _setitem_single_column(self, loc: int, value, plane_indexer) -> None:
 
         is_full_setter = com.is_null_slice(pi) or com.is_full_slice(pi, len(self.obj))
 
-        if is_full_setter:
+        is_null_setter = com.is_empty_slice(pi) or is_array_like(pi) and len(pi) == 0
+
+        if is_null_setter:
+            # no-op, don't cast dtype later
+            return
+
+        elif is_full_setter:
 
             try:
                 self.obj._mgr.column_setitem(
diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py
index 387272eb807e8..f214ade0a31aa 100644
--- a/pandas/tests/series/indexing/test_indexing.py
+++ b/pandas/tests/series/indexing/test_indexing.py
@@ -382,6 +382,16 @@ def test_getitem_bool_int_key():
         ser.loc[0]
 
 
+@pytest.mark.parametrize("val", [{}, {"b": "x"}])
+@pytest.mark.parametrize("indexer", [[], [False, False], slice(0, -1), np.array([])])
+def test_setitem_empty_indexer(indexer, val):
+    # GH#45981
+    df = DataFrame({"a": [1, 2], **val})
+    expected = df.copy()
+    df.loc[indexer] = 1.5
+    tm.assert_frame_equal(df, expected)
+
+
 class TestDeprecatedIndexers:
     @pytest.mark.parametrize("key", [{1}, {1: 1}])
     def test_getitem_dict_and_set_deprecated(self, key):