diff --git a/panel/config.py b/panel/config.py
index 34f34df8ee..a1909cbf3e 100644
--- a/panel/config.py
+++ b/panel/config.py
@@ -798,7 +798,7 @@ def __call__(self, *args, **params):
                 backend = hv.Store.current_backend
             else:
                 backend = 'bokeh'
-            if hasattr(hv.Store, 'set_current_backend'):
+            if not loaded or (loaded and backend != hv.Store.current_backend) and hasattr(hv.Store, 'set_current_backend'):
                 hv.Store.set_current_backend(backend)
             else:
                 hv.Store.current_backend = backend