Improve autoreload of external packages (#6459)

.github/workflows/test.yaml

@@ -300,8 +300,7 @@ jobs:
       PYTHON_VERSION: ${{ matrix.python-version }}
       SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
     steps:
-      # Add back when this works on Python 3.12
-      # - uses: holoviz-dev/holoviz_tasks/install@v0.1a19
+    # - uses: holoviz-dev/holoviz_tasks/install@v0.1a19
       #   with:
       #     name: core_test_suite
       #     python-version: ${{ matrix.python-version }}
@@ -331,4 +330,4 @@ jobs:
       - name: doit test_unit
         run: |
           # conda activate test-environment
-          pytest panel
+          pytest panel -n logical --dist loadgroup

panel/chat/feed.py

@@ -484,12 +484,11 @@ async def _handle_callback(self, message, loop: asyncio.BaseEventLoop):
             response = await self.callback(*callback_args)
         elif isasyncgenfunction(self.callback):
             response = self.callback(*callback_args)
+        elif isgeneratorfunction(self.callback):
+            response = self._to_async_gen(self.callback(*callback_args))
+            # printing type(response) -> <class 'async_generator'>
         else:
-            if isgeneratorfunction(self.callback):
-                response = self._to_async_gen(self.callback(*callback_args))
-                # printing type(response) -> <class 'async_generator'>
-            else:
-                response = await asyncio.to_thread(self.callback, *callback_args)
+            response = await asyncio.to_thread(self.callback, *callback_args)
         await self._serialize_response(response)
 
     async def _prepare_response(self, _) -> None:

panel/command/serve.py

@@ -369,7 +369,7 @@ def customize_kwargs(self, args, server_kwargs):
             argvs = {f: args.args for f in files}
             applications = build_single_handler_applications(files, argvs)
             if args.autoreload:
-                with record_modules():
+                with record_modules(list(applications.values())):
                     self.warm_applications(
                         applications, args.reuse_sessions
                     )

panel/io/application.py

@@ -0,0 +1,93 @@
+"""
+Extensions for Bokeh application handling.
+"""
+from __future__ import annotations
+
+import logging
+import os
+
+from functools import partial
+from typing import TYPE_CHECKING
+
+import bokeh.command.util
+
+from bokeh.application import Application as BkApplication
+from bokeh.application.handlers.directory import DirectoryHandler
+
+from ..config import config
+from .document import _destroy_document
+from .handlers import MarkdownHandler, NotebookHandler, ScriptHandler
+from .logging import LOG_SESSION_DESTROYED, LOG_SESSION_LAUNCHING
+from .state import set_curdoc, state
+
+if TYPE_CHECKING:
+    from bokeh.application.handlers.handler import Handler
+
+log = logging.getLogger('panel.io.application')
+
+
+class Application(BkApplication):
+    """
+    Extends Bokeh Application with ability to add global session
+    creation callbacks, support for the admin dashboard and the
+    ability to globally define a template.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._admin = kwargs.pop('admin', None)
+        super().__init__(*args, **kwargs)
+
+    async def on_session_created(self, session_context):
+        with set_curdoc(session_context._document):
+            if self._admin is not None:
+                config._admin = self._admin
+            for cb in state._on_session_created_internal+state._on_session_created:
+                cb(session_context)
+        await super().on_session_created(session_context)
+
+    def initialize_document(self, doc):
+        log.info(LOG_SESSION_LAUNCHING, id(doc))
+        super().initialize_document(doc)
+        if doc in state._templates and doc not in state._templates[doc]._documents:
+            template = state._templates[doc]
+            with set_curdoc(doc):
+                template.server_doc(title=template.title, location=True, doc=doc)
+        def _log_session_destroyed(session_context):
+            log.info(LOG_SESSION_DESTROYED, id(doc))
+        doc.destroy = partial(_destroy_document, doc) # type: ignore
+        doc.on_event('document_ready', partial(state._schedule_on_load, doc))
+        doc.on_session_destroyed(_log_session_destroyed)
+
+bokeh.command.util.Application = Application # type: ignore
+
+
+def build_single_handler_application(path, argv=None):
+    argv = argv or []
+    path = os.path.abspath(os.path.expanduser(path))
+    handler: Handler
+
+    # There are certainly race conditions here if the file/directory is deleted
+    # in between the isdir/isfile tests and subsequent code. But it would be a
+    # failure if they were not there to begin with, too (just a different error)
+    if os.path.isdir(path):
+        handler = DirectoryHandler(filename=path, argv=argv)
+    elif os.path.isfile(path):
+        if path.endswith(".ipynb"):
+            handler = NotebookHandler(filename=path, argv=argv)
+        elif path.endswith(".md"):
+            handler = MarkdownHandler(filename=path, argv=argv)
+        elif path.endswith(".py"):
+            handler = ScriptHandler(filename=path, argv=argv)
+        else:
+            raise ValueError(f"Expected a '.py' script or '.ipynb' notebook, got: {path!r}" % path)
+    else:
+        raise ValueError(f"Path for Bokeh server application does not exist: {path}")
+
+    if handler.failed:
+        raise RuntimeError(f"Error loading {path}:\n\n{handler.error}\n{handler.error_detail} ")
+
+    application = Application(handler)
+
+    return application
+
+bokeh.command.util.build_single_handler_application = build_single_handler_application

panel/io/convert.py

@@ -24,7 +24,7 @@
 
 from .. import __version__, config
 from ..util import base_version, escape
-from .handlers import build_single_handler_application
+from .application import build_single_handler_application
 from .loading import LOADING_INDICATOR_CSS_CLASS
 from .mime_render import find_requirements
 from .resources import (

panel/io/document.py

@@ -3,10 +3,12 @@
 import asyncio
 import dataclasses
 import datetime as dt
+import gc
 import inspect
 import json
 import logging
 import threading
+import weakref
 
 from contextlib import contextmanager
 from functools import partial, wraps
@@ -173,6 +175,44 @@ async def _dispatch_msgs(doc, msgs):
     await asyncio.sleep(0.01)
     _dispatch_write_task(doc, _dispatch_msgs, doc, remaining)
 
+def _destroy_document(self, session):
+    """
+    Override for Document.destroy() without calling gc.collect directly.
+    The gc.collect() call is scheduled as a task, ensuring that when
+    multiple documents are destroyed in quick succession we do not
+    schedule excessive garbage collection.
+    """
+    if session is not None:
+        self.remove_on_change(session)
+
+    del self._roots
+    del self._theme
+    del self._template
+    self._session_context = None
+
+    self.callbacks.destroy()
+    self.models.destroy()
+    self.modules.destroy()
+
+    # Clear periodic callbacks
+    for cb in state._periodic.get(self, []):
+        cb.stop()
+
+    # Clean up pn.state to avoid tasks getting executed on dead session
+    for attr in dir(state):
+        # _param_watchers is deprecated in Param 2.0 and will raise a warning
+        if not attr.startswith('_') or attr == "_param_watchers":
+            continue
+        state_obj = getattr(state, attr)
+        if isinstance(state_obj, weakref.WeakKeyDictionary) and self in state_obj:
+            del state_obj[self]
+
+    # Schedule GC
+    at = dt.datetime.now() + dt.timedelta(seconds=5)
+    state.schedule_task('gc.collect', gc.collect, at=at)
+
+    del self.destroy
+
 #---------------------------------------------------------------------
 # Public API
 #---------------------------------------------------------------------