diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 6d368e473b..1e73eb3485 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -1062,7 +1062,7 @@ T cast(const handle &handle) {
 template <typename T,
           detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value, int> = 0>
 T cast(const handle &handle) {
-    return handle.ptr();
+    return handle.inc_ref().ptr();
 }
 
 // C++ type -> py::object
diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h
index 5d538d20ed..3b6a7bed75 100644
--- a/include/pybind11/type_caster_pyobject_ptr.h
+++ b/include/pybind11/type_caster_pyobject_ptr.h
@@ -35,8 +35,12 @@ class type_caster<PyObject> {
         if (policy == return_value_policy::take_ownership) {
             return src;
         }
-        Py_INCREF(src);
-        return src;
+        if (policy == return_value_policy::reference
+            || policy == return_value_policy::automatic_reference) {
+            return handle(src).inc_ref();
+        }
+        pybind11_fail("type_caster<PyObject>::cast(): unsupported return_value_policy: "
+                      + std::to_string(static_cast<int>(policy)));
     }
 
     bool load(handle src, bool) {
diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp
index 71fbd383d9..4e476d34ea 100644
--- a/tests/test_type_caster_pyobject_ptr.cpp
+++ b/tests/test_type_caster_pyobject_ptr.cpp
@@ -21,14 +21,16 @@ TEST_SUBMODULE(type_caster_pyobject_ptr, m) {
     m.def("pass_pyobject_ptr", [](PyObject *obj) { return bool(PyTuple_CheckExact(obj)); });
 
     m.def("call_callback_with_object_return",
-          [](const std::function<py::object(int mode)> &cb, int mode) { return cb(mode); });
-    m.def("call_callback_with_handle_return",
-          [](const std::function<py::handle(int mode)> &cb, int mode) { return cb(mode); });
-    //
-    m.def("call_callback_with_pyobject_ptr_return",
-          [](const std::function<PyObject *(int mode)> &cb, int mode) { return cb(mode); });
-    m.def("call_callback_with_pyobject_ptr_arg",
-          [](const std::function<bool(PyObject *)> &cb, py::handle obj) { return cb(obj.ptr()); });
+          [](const std::function<py::object(int)> &cb, int value) { return cb(value); });
+    m.def(
+        "call_callback_with_pyobject_ptr_return",
+        [](const std::function<PyObject *(int)> &cb, int value) { return cb(value); },
+        py::return_value_policy::take_ownership);
+    m.def(
+        "call_callback_with_pyobject_ptr_arg",
+        [](const std::function<bool(PyObject *)> &cb, py::handle obj) { return cb(obj.ptr()); },
+        py::arg("cb"), // This triggers return_value_policy::automatic_reference
+        py::arg("obj"));
 
     m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) {
         if (set_error) {
diff --git a/tests/test_type_caster_pyobject_ptr.py b/tests/test_type_caster_pyobject_ptr.py
index 699349523d..6271e92012 100644
--- a/tests/test_type_caster_pyobject_ptr.py
+++ b/tests/test_type_caster_pyobject_ptr.py
@@ -21,26 +21,29 @@ def test_pass_pyobject_ptr():
     assert not m.pass_pyobject_ptr({})
 
 
+class ValueHolder:
+    def __init__(self, value):
+        self.value = value
+
+
 @pytest.mark.parametrize(
     "call_callback",
     [
         m.call_callback_with_object_return,
-        m.call_callback_with_handle_return,
         m.call_callback_with_pyobject_ptr_return,
     ],
 )
 def test_call_callback_with_object_return(call_callback):
-    def cb(mode):
-        if mode == 0:
-            return 10
-        if mode == 1:
-            return "One"
-        raise NotImplementedError(f"Unknown mode: {mode}")
-
-    assert call_callback(cb, 0) == 10
-    assert call_callback(cb, 1) == "One"
-    with pytest.raises(NotImplementedError, match="Unknown mode: 2"):
-        call_callback(cb, 2)
+    def cb(value):
+        if value < 0:
+            raise ValueError("Raised from cb")
+        # Return a temporary user-defined object, to maximize sensitivity of this test.
+        return ValueHolder(1000 - value)
+
+    assert call_callback(cb, 287).value == 713
+
+    with pytest.raises(ValueError, match="^Raised from cb$"):
+        call_callback(cb, -1)
 
 
 def test_call_callback_with_pyobject_ptr_arg():