Remove direct usage of tornado

.github/workflows/downstream.yml

@@ -31,15 +31,30 @@ jobs:
           package_name: nbclient
           env_values: IPYKERNEL_CELL_NAME=\<IPY-INPUT\>
 
-  papermill:
+  # papermill:
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 15
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+  #     - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1
+  #       with:
+  #         package_name: papermill
+
+  pytest_jupyter:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
-      - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
-      - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Run Test
+        uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1
         with:
-          package_name: papermill
+          package_name: pytest_jupyter
+          package_spec: pip install -e ".[test,client,server]"
 
   nbconvert:
     runs-on: ubuntu-latest
@@ -61,6 +76,7 @@ jobs:
       - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1
         with:
           package_name: jupyter_server
+          test_command: pytest -vv -ras -W default --durations 10 --color=yes
 
   jupyter_kernel_test:
     runs-on: ubuntu-latest
@@ -131,7 +147,8 @@ jobs:
     needs:
       - ipykernel
       - nbclient
-      - papermill
+      #- papermill
+      - pytest_jupyter
       - nbconvert
       - jupyter_server
       - jupyter_kernel_test

.pre-commit-config.yaml

@@ -29,6 +29,8 @@ repos:
     rev: 0.7.17
     hooks:
       - id: mdformat
+        additional_dependencies:
+          [mdformat-gfm, mdformat-frontmatter, mdformat-footnote]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: "v4.0.0-alpha.8"

docs/migration.md

@@ -1,5 +1,41 @@
 # Migration Guide
 
+## Jupyter Client 8.0 to 9.0
+
+Overall changes: removed direct usages of `tornado` in favor of `asyncio` loops.
+We are still using `zmq.eventloop.zmqstream.ZMQStream`, which uses `tornado`
+loops internally.
+
+### API Changes
+
+The following `loop` properties are now asyncio loops:
+
+- `jupyter_client.ioloop.manager.IOLoopKernelManager.loop`
+- `jupyter_client.ioloop.manager.AsyncIOLoopKernelManager.loop`
+- `jupyter_client.ioloop.restarter.IOLoopKernelRestarter.loop`
+- `jupyter_client.ioloop.restarter.AsyncIOLoopKernelRestarter.loop`
+- `jupyter_client.threaded.ThreadedZMQSocketChannel.loop`
+- `jupyter_client.threaded.IOLoopThread.ioloop`
+- `jupyter_client.threaded.ThreadedKernelClient.ioloop`
+
+The `jupyter_client.kernelapp.KernelApp` class now subclasses
+`jupyter_core.application.JupyterAsyncApp`, and performs its initialization
+from within an asyncio loop.
+
+The following function was added as a shim to the `jupyter_core` utility:
+
+- `jupyter_client.utils.ensure_event_loop`
+
+## Jupyter Client 7.0 to 8.0
+
+The main goal of this release was to improve handling of asyncio and remove
+the need for `nest_asyncio`.
+
+### API Changes
+
+- `jupyter_client.asynchronous.client.AsyncKernelClient.context` is now a `zmq.asyncio.Context`
+- `jupyter_client.asynchronous.client.AsyncKernelClient.*_channel_class` are now instances of `jupyter_client.channels.AsyncZMQSocketChannel`
+
 ## Jupyter Client 6.0 to 7.0
 
 ### API Changes

jupyter_client/client.py

@@ -344,7 +344,7 @@ def channels_running(self) -> bool:
             or (self._control_channel and self.control_channel.is_alive())
         )
 
-    ioloop = None  # Overridden in subclasses that use pyzmq event loop
+    ioloop = None  # Overridden in subclasses that use asyncio event loop
 
     @property
     def shell_channel(self) -> t.Any:

jupyter_client/ioloop/manager.py

@@ -1,18 +1,20 @@
-"""A kernel manager with a tornado IOLoop"""
+"""A kernel manager with an asyncio IOLoop"""
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+import asyncio
 import typing as t
 
 import zmq
-from tornado import ioloop
+import zmq.asyncio
+from jupyter_core.utils import ensure_event_loop
 from traitlets import Instance, Type
 from zmq.eventloop.zmqstream import ZMQStream
 
 from ..manager import AsyncKernelManager, KernelManager
 from .restarter import AsyncIOLoopKernelRestarter, IOLoopKernelRestarter
 
 
-def as_zmqstream(f: t.Any) -> t.Callable:
+def as_zmqstream(f: t.Any) -> t.Callable[..., ZMQStream]:
     """Convert a socket to a zmq stream."""
 
     def wrapped(self: t.Any, *args: t.Any, **kwargs: t.Any) -> t.Any:
@@ -27,18 +29,18 @@ def wrapped(self: t.Any, *args: t.Any, **kwargs: t.Any) -> t.Any:
             if save_socket_class:
                 # restore default socket class
                 self.context._socket_class = save_socket_class
-        return ZMQStream(socket, self.loop)
+        return ZMQStream(socket)
 
     return wrapped
 
 
 class IOLoopKernelManager(KernelManager):
     """An io loop kernel manager."""
 
-    loop = Instance("tornado.ioloop.IOLoop")
+    loop = Instance(asyncio.AbstractEventLoop)  # type:ignore[type-abstract]
 
-    def _loop_default(self) -> ioloop.IOLoop:
-        return ioloop.IOLoop.current()
+    def _loop_default(self) -> asyncio.AbstractEventLoop:
+        return ensure_event_loop()
 
     restarter_class = Type(
         default_value=IOLoopKernelRestarter,
@@ -57,7 +59,7 @@ def start_restarter(self) -> None:
         if self.autorestart and self.has_kernel:
             if self._restarter is None:
                 self._restarter = self.restarter_class(
-                    kernel_manager=self, loop=self.loop, parent=self, log=self.log
+                    kernel_manager=self, parent=self, log=self.log
                 )
             self._restarter.start()
 
@@ -66,20 +68,20 @@ def stop_restarter(self) -> None:
         if self.autorestart and self._restarter is not None:
             self._restarter.stop()
 
-    connect_shell = as_zmqstream(KernelManager.connect_shell)
-    connect_control = as_zmqstream(KernelManager.connect_control)
-    connect_iopub = as_zmqstream(KernelManager.connect_iopub)
-    connect_stdin = as_zmqstream(KernelManager.connect_stdin)
-    connect_hb = as_zmqstream(KernelManager.connect_hb)
+    connect_shell = as_zmqstream(KernelManager.connect_shell)  # type:ignore[assignment]
+    connect_control = as_zmqstream(KernelManager.connect_control)  # type:ignore[assignment]
+    connect_iopub = as_zmqstream(KernelManager.connect_iopub)  # type:ignore[assignment]
+    connect_stdin = as_zmqstream(KernelManager.connect_stdin)  # type:ignore[assignment]
+    connect_hb = as_zmqstream(KernelManager.connect_hb)  # type:ignore[assignment]
 
 
 class AsyncIOLoopKernelManager(AsyncKernelManager):
     """An async ioloop kernel manager."""
 
-    loop = Instance("tornado.ioloop.IOLoop")
+    loop = Instance(asyncio.AbstractEventLoop)  # type:ignore[type-abstract]
 
-    def _loop_default(self) -> ioloop.IOLoop:
-        return ioloop.IOLoop.current()
+    def _loop_default(self) -> asyncio.AbstractEventLoop:
+        return ensure_event_loop()
 
     restarter_class = Type(
         default_value=AsyncIOLoopKernelRestarter,
@@ -100,7 +102,7 @@ def start_restarter(self) -> None:
         if self.autorestart and self.has_kernel:
             if self._restarter is None:
                 self._restarter = self.restarter_class(
-                    kernel_manager=self, loop=self.loop, parent=self, log=self.log
+                    kernel_manager=self, parent=self, log=self.log
                 )
             self._restarter.start()
 
@@ -109,8 +111,8 @@ def stop_restarter(self) -> None:
         if self.autorestart and self._restarter is not None:
             self._restarter.stop()
 
-    connect_shell = as_zmqstream(AsyncKernelManager.connect_shell)
-    connect_control = as_zmqstream(AsyncKernelManager.connect_control)
-    connect_iopub = as_zmqstream(AsyncKernelManager.connect_iopub)
-    connect_stdin = as_zmqstream(AsyncKernelManager.connect_stdin)
-    connect_hb = as_zmqstream(AsyncKernelManager.connect_hb)
+    connect_shell = as_zmqstream(AsyncKernelManager.connect_shell)  # type:ignore[assignment]
+    connect_control = as_zmqstream(AsyncKernelManager.connect_control)  # type:ignore[assignment]
+    connect_iopub = as_zmqstream(AsyncKernelManager.connect_iopub)  # type:ignore[assignment]
+    connect_stdin = as_zmqstream(AsyncKernelManager.connect_stdin)  # type:ignore[assignment]
+    connect_hb = as_zmqstream(AsyncKernelManager.connect_hb)  # type:ignore[assignment]

jupyter_client/ioloop/restarter.py

@@ -5,10 +5,12 @@
 """
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+from __future__ import annotations
+
+import asyncio
 import time
-import warnings
-from typing import Any
 
+from jupyter_core.utils import ensure_async, ensure_event_loop
 from traitlets import Instance
 
 from ..restarter import KernelRestarter
@@ -17,36 +19,32 @@
 class IOLoopKernelRestarter(KernelRestarter):
     """Monitor and autorestart a kernel."""
 
-    loop = Instance("tornado.ioloop.IOLoop")
-
-    def _loop_default(self) -> Any:
-        warnings.warn(
-            "IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2",
-            DeprecationWarning,
-            stacklevel=4,
-        )
-        from tornado import ioloop
+    _poll_task: asyncio.Task | None = None
+    _running = False
 
-        return ioloop.IOLoop.current()
+    loop = Instance(asyncio.AbstractEventLoop)  # type:ignore[type-abstract]
 
-    _pcallback = None
+    def _loop_default(self) -> asyncio.AbstractEventLoop:
+        return ensure_event_loop()
 
     def start(self) -> None:
         """Start the polling of the kernel."""
-        if self._pcallback is None:
-            from tornado.ioloop import PeriodicCallback
+        if not self._poll_task:
+            assert self.parent is not None
+            assert isinstance(self.parent.loop, asyncio.AbstractEventLoop)
+            self._poll_task = self.parent.loop.create_task(self._poll_loop())
+            self._running = True
 
-            self._pcallback = PeriodicCallback(
-                self.poll,
-                1000 * self.time_to_dead,
-            )
-            self._pcallback.start()
+    async def _poll_loop(self) -> None:
+        while self._running:
+            await ensure_async(self.poll())  # type:ignore[func-returns-value]
+            await asyncio.sleep(0.01)
 
     def stop(self) -> None:
         """Stop the kernel polling."""
-        if self._pcallback is not None:
-            self._pcallback.stop()
-            self._pcallback = None
+        if self._poll_task is not None:
+            self._poll_task = None
+            self._running = False
 
 
 class AsyncIOLoopKernelRestarter(IOLoopKernelRestarter):

jupyter_client/kernelapp.py

@@ -1,25 +1,26 @@
 """An application to launch a kernel by name in a local subprocess."""
+import asyncio
+import functools
 import os
 import signal
 import typing as t
 import uuid
 
-from jupyter_core.application import JupyterApp, base_flags
-from tornado.ioloop import IOLoop
+from jupyter_core.application import JupyterAsyncApp, base_flags
 from traitlets import Unicode
 
 from . import __version__
 from .kernelspec import NATIVE_KERNEL_NAME, KernelSpecManager
-from .manager import KernelManager
+from .manager import AsyncKernelManager
 
 
-class KernelApp(JupyterApp):
+class KernelApp(JupyterAsyncApp):
     """Launch a kernel by name in a local subprocess."""
 
     version = __version__
     description = "Run a kernel locally in a subprocess"
 
-    classes = [KernelManager, KernelSpecManager]
+    classes = [AsyncKernelManager, KernelSpecManager]
 
     aliases = {
         "kernel": "KernelApp.kernel_name",
@@ -31,35 +32,29 @@ class KernelApp(JupyterApp):
         config=True
     )
 
-    def initialize(self, argv: t.Union[str, t.Sequence[str], None] = None) -> None:
+    async def initialize_async(self, argv: t.Union[str, t.Sequence[str], None] = None) -> None:
         """Initialize the application."""
-        super().initialize(argv)
-
         cf_basename = "kernel-%s.json" % uuid.uuid4()
         self.config.setdefault("KernelManager", {}).setdefault(
             "connection_file", os.path.join(self.runtime_dir, cf_basename)
         )
-        self.km = KernelManager(kernel_name=self.kernel_name, config=self.config)
-
-        self.loop = IOLoop.current()
-        self.loop.add_callback(self._record_started)
+        self.km = AsyncKernelManager(kernel_name=self.kernel_name, config=self.config)
+        self._record_started()
+        self._stopped_fut: asyncio.Future[int] = asyncio.Future()
+        self._running = None
 
     def setup_signals(self) -> None:
         """Shutdown on SIGTERM or SIGINT (Ctrl-C)"""
         if os.name == "nt":
             return
 
-        def shutdown_handler(signo: int, frame: t.Any) -> None:
-            self.loop.add_callback_from_signal(self.shutdown, signo)
-
-        for sig in [signal.SIGTERM, signal.SIGINT]:
-            signal.signal(sig, shutdown_handler)
+        loop = asyncio.get_running_loop()
+        for signo in [signal.SIGTERM, signal.SIGINT]:
+            loop.add_signal_handler(signo, functools.partial(self.shutdown, signo))
 
     def shutdown(self, signo: int) -> None:
         """Shut down the application."""
-        self.log.info("Shutting down on signal %d", signo)
-        self.km.shutdown_kernel()
-        self.loop.stop()
+        self._stopped_fut.set_result(signo)
 
     def log_connection_info(self) -> None:
         """Log the connection info for the kernel."""
@@ -77,16 +72,18 @@ def _record_started(self) -> None:
             with open(fn, "wb"):
                 pass
 
-    def start(self) -> None:
-        """Start the application."""
+    async def start_async(self) -> None:
         self.log.info("Starting kernel %r", self.kernel_name)
+        km = self.km
         try:
-            self.km.start_kernel()
-            self.log_connection_info()
             self.setup_signals()
-            self.loop.start()
+            self.log_connection_info()
+            await km.start_kernel()
+            stopped_sig = await self._stopped_fut
+            self.log.info("Shutting down on signal %d", stopped_sig)
+            await km.shutdown_kernel()
         finally:
-            self.km.cleanup_resources()
+            await km.cleanup_resources()
 
 
 main = KernelApp.launch_instance