diff --git a/panel/chat/feed.py b/panel/chat/feed.py
index a6c48b2c43..d866a8b289 100644
--- a/panel/chat/feed.py
+++ b/panel/chat/feed.py
@@ -471,7 +471,11 @@ async def _prepare_response(self, _) -> None:
         except Exception as e:
             send_kwargs = dict(user="Exception", respond=False)
             if self.callback_exception == "summary":
-                self.send(str(e), **send_kwargs)
+                self.send(
+                    f"Encountered `{e!r}`. "
+                    f"Set `callback_exception='verbose'` to see the full traceback.",
+                    **send_kwargs
+                )
             elif self.callback_exception == "verbose":
                 self.send(f"```python\n{traceback.format_exc()}\n```", **send_kwargs)
             elif self.callback_exception == "ignore":
diff --git a/panel/tests/chat/test_feed.py b/panel/tests/chat/test_feed.py
index 841779ec4a..8d9a08cc5e 100644
--- a/panel/tests/chat/test_feed.py
+++ b/panel/tests/chat/test_feed.py
@@ -633,7 +633,7 @@ def callback(msg, user, instance):
         chat_feed.callback = callback
         chat_feed.callback_exception = "summary"
         chat_feed.send("Message", respond=True)
-        assert chat_feed.objects[-1].object == "division by zero"
+        assert "division by zero" in chat_feed.objects[-1].object
         assert chat_feed.objects[-1].user == "Exception"
 
     def test_callback_exception_traceback(self, chat_feed):