Allow arbitrary contents managers

docs/user_guide/index.md

@@ -19,11 +19,11 @@ and asserting that `jupyter_server_fileid` is enabled.
 
 ## Usage
 
-`jupyter_server_fileid`, by default, constructs a `FileIdManager` instance and
-stores it under `serverapp.settings["file_id_manager"]`. This `FileIdManager`
+`jupyter_server_fileid`, by default, constructs a `LocalFileIdManager` instance and
+stores it under `serverapp.settings["file_id_manager"]`. This `LocalFileIdManager`
 instance is a developer's way of accessing its key methods.
 
-Once you obtain a reference to the `FileIdManager` instance and bind it to some
+Once you obtain a reference to the `LocalFileIdManager` instance and bind it to some
 name, e.g. `fim`, all file ID operations can be performed as methods on that
 object. File ID operations are best illustrated in the following examples.
 

jupyter_server_fileid/extension.py

@@ -1,8 +1,13 @@
 from jupyter_events.logger import EventLogger
 from jupyter_server.extension.application import ExtensionApp
-from traitlets import Type, default
+from jupyter_server.services.contents.filemanager import FileContentsManager
+from traitlets import Type
 
-from jupyter_server_fileid.manager import FileIdManager
+from jupyter_server_fileid.manager import (
+    ArbitraryFileIdManager,
+    BaseFileIdManager,
+    LocalFileIdManager,
+)
 
 from .handlers import FileId2PathHandler, FilePath2IdHandler
 
@@ -12,18 +17,32 @@ class FileIdExtension(ExtensionApp):
     name = "jupyter_server_fileid"
 
     file_id_manager_class = Type(
-        klass=FileIdManager,
-        help="File ID manager instance to use. Defaults to FileIdManager.",
+        klass=BaseFileIdManager,
+        help="""File ID manager instance to use.
+
+        Defaults to:
+        - LocalFileIdManager if contents manager is a FileContentsManager,
+        - ArbitraryFileIdManager otherwise.
+        """,
         config=True,
+        default_value=None,
+        allow_none=True,
     )
 
-    @default("file_id_manager")
-    def _file_id_manager_default(self):
-        self.log.debug("No File ID manager configured. Defaulting to FileIdManager.")
-        return FileIdManager
-
     def initialize_settings(self):
-        self.log.debug(f"Configured File ID manager: {self.file_id_manager_class.__name__}")
+        if self.file_id_manager_class is None:
+            if isinstance(self.settings["contents_manager"], FileContentsManager):
+                self.log.info(
+                    "Contents manager is a FileContentsManager. Defaulting to LocalFileIdManager."
+                )
+                self.file_id_manager_class = LocalFileIdManager
+            else:
+                self.log.info(
+                    "Contents manager is not a FileContentsManager. Defaulting to ArbitraryFileIdManager."
+                )
+                self.file_id_manager_class = ArbitraryFileIdManager
+
+        self.log.info(f"Configured File ID manager: {self.file_id_manager_class.__name__}")
         file_id_manager = self.file_id_manager_class(
             log=self.log, root_dir=self.serverapp.root_dir, config=self.config
         )
@@ -35,15 +54,7 @@ def initialize_settings(self):
 
     def initialize_event_listeners(self):
         file_id_manager = self.settings["file_id_manager"]
-
-        # define event handlers per contents manager action
-        handlers_by_action = {
-            "get": None,
-            "save": lambda data: file_id_manager.save(data["path"]),
-            "rename": lambda data: file_id_manager.move(data["source_path"], data["path"]),
-            "copy": lambda data: file_id_manager.copy(data["source_path"], data["path"]),
-            "delete": lambda data: file_id_manager.delete(data["path"]),
-        }
+        handlers_by_action = file_id_manager.get_handlers_by_action()
 
         async def cm_listener(logger: EventLogger, schema_id: str, data: dict) -> None:
             handler = handlers_by_action[data["action"]]

jupyter_server_fileid/manager.py

@@ -2,7 +2,7 @@
 import sqlite3
 import stat
 import time
-from typing import Optional
+from typing import Any, Callable, Dict, Optional, Union
 
 from jupyter_core.paths import jupyter_data_dir
 from traitlets import Int, TraitError, Unicode, validate
@@ -37,35 +37,174 @@ def wrapped(self, *args, **kwargs):
     return decorator
 
 
-class FileIdManager(LoggingConfigurable):
+class BaseFileIdManager(LoggingConfigurable):
     """
-    Manager that supports tracking files across their lifetime by associating
-    each with a unique file ID, which is maintained across filesystem operations.
-
-    Notes
-    -----
-
-    All private helper methods prefixed with an underscore (except `__init__()`)
-    do NOT commit their SQL statements in a transaction via `self.con.commit()`.
-    This responsibility is delegated to the public method calling them to
-    increase performance. Committing multiple SQL transactions in serial is much
-    slower than committing a single SQL transaction wrapping all SQL statements
-    performed during a method's procedure body.
+    Base class for File ID manager implementations. All File ID
+    managers should inherit from this class.
     """
 
     root_dir = Unicode(
-        help=("The root being served by Jupyter server. Must be an absolute path."), config=True
+        help=("The root directory being served by Jupyter server. Must be an absolute path."),
+        config=False,
     )
 
     db_path = Unicode(
         default_value=default_db_path,
         help=(
-            "The path of the DB file used by `FileIdManager`. "
+            "The path of the DB file used by File ID manager implementations. "
             "Defaults to `jupyter_data_dir()/file_id_manager.db`."
         ),
         config=True,
     )
 
+    @validate("root_dir", "db_path")
+    def _validate_abspath_traits(self, proposal):
+        if proposal["value"] is None:
+            raise TraitError(f"FileIdManager : {proposal['trait'].name} must not be None")
+        if not os.path.isabs(proposal["value"]):
+            raise TraitError(f"FileIdManager : {proposal['trait'].name} must be an absolute path")
+        return proposal["value"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def index(self, path: str) -> Union[int, str, None]:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def get_id(self, path: str) -> Union[int, str, None]:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def get_path(self, id: Union[int, str]) -> Union[int, str, None]:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def move(self, old_path: str, new_path: str) -> Union[int, str, None]:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def copy(self, from_path: str, to_path: str) -> Union[int, str, None]:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def delete(self, path: str) -> None:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def save(self, path: str) -> None:
+        raise NotImplementedError("must be implemented by subclass")
+
+    def get_handlers_by_action(self) -> Dict[str, Optional[Callable[[Dict[str, Any]], Any]]]:
+        """Returns a dictionary whose keys are contents manager event actions
+        and whose values are callables invoked upon receipt of an event of the
+        same action. The callable accepts the body of the event as its only
+        argument. To ignore an event action, set the value to `None`."""
+        raise NotImplementedError("must be implemented by subclass")
+
+
+class ArbitraryFileIdManager(BaseFileIdManager):
+    """
+    File ID manager that works on arbitrary filesystems. Each file is assigned a
+    unique ID. The path is only updated upon calling `move()`, `copy()`, or
+    `delete()`, e.g. upon receipt of contents manager events emitted by Jupyter
+    Server 2.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # pass args and kwargs to parent Configurable
+        super().__init__(*args, **kwargs)
+        # initialize instance attrs
+        self._update_cursor = False
+        # initialize connection with db
+        self.log.info(f"ArbitraryFileIdManager : Configured root dir: {self.root_dir}")
+        self.log.info(f"ArbitraryFileIdManager : Configured database path: {self.db_path}")
+        self.con = sqlite3.connect(self.db_path)
+        self.log.info("ArbitraryFileIdManager : Successfully connected to database file.")
+        self.log.info("ArbitraryFileIdManager : Creating File ID tables and indices.")
+        # do not allow reads to block writes. required when using multiple processes
+        self.con.execute("PRAGMA journal_mode = WAL")
+        self.con.execute(
+            "CREATE TABLE IF NOT EXISTS Files("
+            "id INTEGER PRIMARY KEY AUTOINCREMENT, "
+            "path TEXT NOT NULL UNIQUE"
+            ")"
+        )
+        self.con.execute("CREATE INDEX IF NOT EXISTS ix_Files_path ON Files (path)")
+        self.con.commit()
+
+    def index(self, path: str) -> int:
+        row = self.con.execute("SELECT id FROM Files WHERE path = ?", (path,)).fetchone()
+        existing_id = row and row[0]
+
+        if existing_id:
+            return existing_id
+
+        # create new record
+        cursor = self.con.execute("INSERT INTO Files (path) VALUES (?)", (path,))
+        self.con.commit()
+        return cursor.lastrowid  # type:ignore
+
+    def get_id(self, path: str) -> Optional[int]:
+        row = self.con.execute("SELECT id FROM Files WHERE path = ?", (path,)).fetchone()
+        return row and row[0]
+
+    def get_path(self, id: Union[int, str]) -> Optional[int]:
+        row = self.con.execute("SELECT path FROM Files WHERE id = ?", (id,)).fetchone()
+        return row and row[0]
+
+    def move(self, old_path: str, new_path: str) -> None:
+        row = self.con.execute("SELECT id FROM Files WHERE path = ?", (old_path,)).fetchone()
+        id = row and row[0]
+
+        if id:
+            self.con.execute("UPDATE Files SET path = ? WHERE path = ?", (new_path, old_path))
+        else:
+            cursor = self.con.execute("INSERT INTO Files (path) VALUES (?)", (new_path,))
+            id = cursor.lastrowid
+
+        self.con.commit()
+        return id
+
+    def copy(self, from_path: str, to_path: str) -> Optional[int]:
+        cursor = self.con.execute("INSERT INTO Files (path) VALUES (?)", (to_path,))
+        self.con.commit()
+        return cursor.lastrowid
+
+    def delete(self, path: str) -> None:
+        self.con.execute("DELETE FROM Files WHERE path = ?", (path,))
+        self.con.commit()
+
+    def save(self, path: str) -> None:
+        return
+
+    def get_handlers_by_action(self) -> Dict[str, Optional[Callable[[Dict[str, Any]], Any]]]:
+        return {
+            "get": None,
+            "save": None,
+            "rename": lambda data: self.move(data["source_path"], data["path"]),
+            "copy": lambda data: self.copy(data["source_path"], data["path"]),
+            "delete": lambda data: self.delete(data["path"]),
+        }
+
+    def __del__(self):
+        """Cleans up `ArbitraryFileIdManager` by committing any pending
+        transactions and closing the connection."""
+        if hasattr(self, "con"):
+            self.con.commit()
+            self.con.close()
+
+
+class LocalFileIdManager(BaseFileIdManager):
+    """
+    File ID manager that supports tracking files in local filesystems by
+    associating each with a unique file ID, which is maintained across
+    filesystem operations.
+
+    Notes
+    -----
+    All private helper methods prefixed with an underscore (except `__init__()`)
+    do NOT commit their SQL statements in a transaction via `self.con.commit()`.
+    This responsibility is delegated to the public method calling them to
+    increase performance. Committing multiple SQL transactions in serial is much
+    slower than committing a single SQL transaction wrapping all SQL statements
+    performed during a method's procedure body.
+    """
+
     autosync_interval = Int(
         default_value=5,
         help=(
@@ -84,11 +223,11 @@ def __init__(self, *args, **kwargs):
         self._update_cursor = False
         self._last_sync = 0.0
         # initialize connection with db
-        self.log.info(f"FileIdManager : Configured root dir: {self.root_dir}")
-        self.log.info(f"FileIdManager : Configured database path: {self.db_path}")
+        self.log.info(f"LocalFileIdManager : Configured root dir: {self.root_dir}")
+        self.log.info(f"LocalFileIdManager : Configured database path: {self.db_path}")
         self.con = sqlite3.connect(self.db_path)
-        self.log.info("FileIdManager : Successfully connected to database file.")
-        self.log.info("FileIdManager : Creating File ID tables and indices.")
+        self.log.info("LocalFileIdManager : Successfully connected to database file.")
+        self.log.info("LocalFileIdManager : Creating File ID tables and indices.")
         # do not allow reads to block writes. required when using multiple processes
         self.con.execute("PRAGMA journal_mode = WAL")
         self.con.execute(
@@ -109,14 +248,6 @@ def __init__(self, *args, **kwargs):
         self.con.execute("CREATE INDEX IF NOT EXISTS ix_Files_is_dir ON Files (is_dir)")
         self.con.commit()
 
-    @validate("root_dir", "db_path")
-    def _validate_abspath_traits(self, proposal):
-        if proposal["value"] is None:
-            raise TraitError(f"FileIdManager : {proposal['trait'].name} must not be None")
-        if not os.path.isabs(proposal["value"]):
-            raise TraitError(f"FileIdManager : {proposal['trait'].name} must be an absolute path")
-        return self._normalize_path(proposal["value"])
-
     def _index_all(self):
         """Recursively indexes all directories under the server root."""
         self._index_dir_recursively(self.root_dir, self._stat(self.root_dir))
@@ -598,8 +729,17 @@ def save(self, path):
         self._update(id, stat_info)
         self.con.commit()
 
+    def get_handlers_by_action(self) -> Dict[str, Optional[Callable[[Dict[str, Any]], Any]]]:
+        return {
+            "get": None,
+            "save": lambda data: self.save(data["path"]),
+            "rename": lambda data: self.move(data["source_path"], data["path"]),
+            "copy": lambda data: self.copy(data["source_path"], data["path"]),
+            "delete": lambda data: self.delete(data["path"]),
+        }
+
     def __del__(self):
-        """Cleans up `FileIdManager` by committing any pending transactions and
+        """Cleans up `LocalFileIdManager` by committing any pending transactions and
         closing the connection."""
         if hasattr(self, "con"):
             self.con.commit()

jupyter_server_fileid/pytest_plugin.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from jupyter_server_fileid.manager import FileIdManager
+from jupyter_server_fileid.manager import ArbitraryFileIdManager, LocalFileIdManager
 
 
 @pytest.fixture
@@ -30,8 +30,8 @@ def delete_fid_db(fid_db_path):
 
 @pytest.fixture
 def fid_manager(fid_db_path, jp_root_dir):
-    """Fixture returning a test-configured instance of `FileIdManager`."""
-    fid_manager = FileIdManager(db_path=fid_db_path, root_dir=str(jp_root_dir))
+    """Fixture returning a test-configured instance of `LocalFileIdManager`."""
+    fid_manager = LocalFileIdManager(db_path=fid_db_path, root_dir=str(jp_root_dir))
     # disable journal so no temp journal file is created under `tmp_path`.
     # reduces test flakiness since sometimes journal file has same ino and
     # crtime as a deleted file, so FID manager detects it wrongly as a move
@@ -40,6 +40,17 @@ def fid_manager(fid_db_path, jp_root_dir):
     return fid_manager
 
 
+@pytest.fixture(params=["local", "arbitrary"])
+def any_fid_manager(request, fid_db_path, jp_root_dir):
+    """Parametrized fixture that runs the test with each of the default File ID
+    manager implementations."""
+    class_by_param = {"local": LocalFileIdManager, "arbitrary": ArbitraryFileIdManager}
+
+    fid_manager = class_by_param[request.param](db_path=fid_db_path, root_dir=str(jp_root_dir))
+    fid_manager.con.execute("PRAGMA journal_mode = OFF")  # type: ignore[attr-defined]
+    return fid_manager
+
+
 @pytest.fixture
 def fs_helpers(jp_root_dir):
     class FsHelpers: