⬆️ UPGRADE: sqlalchemy v1.4 (v2 API)

.github/workflows/ci.yml

@@ -46,6 +46,8 @@ jobs:
       # No need to run the benchmarks, they will run in a different workflow
       # Also, run in very verbose mode so if there is an error we get a complete diff
       run: pytest -vv --cov=disk_objectstore --benchmark-skip
+      env:
+        SQLALCHEMY_WARN_20: 1
     - name: Create xml coverage
       run: coverage xml
     - name: Upload coverage to Codecov

.pre-commit-config.yaml

@@ -60,7 +60,7 @@ repos:
     hooks:
     - id: pylint
       additional_dependencies:
-      - sqlalchemy<1.4
+      - sqlalchemy==1.4.22
       - click==8.0.1
       - memory-profiler==0.58.0
       - profilehooks==1.12.0
@@ -71,5 +71,7 @@ repos:
     rev: v0.910
     hooks:
     - id: mypy
-      additional_dependencies: [typing-extensions]
+      additional_dependencies:
+      - "sqlalchemy[mypy]==1.4.22"
+      - typing-extensions
       files: ^(disk_objectstore/.*py)$

disk_objectstore/database.py

@@ -0,0 +1,94 @@
+"""Models for the container index file (SQLite DB)."""
+import os
+from typing import Optional
+
+from sqlalchemy import Boolean, Column, Integer, String, create_engine, event
+from sqlalchemy.orm import declarative_base, sessionmaker
+from sqlalchemy.orm.session import Session
+from sqlalchemy.sql.expression import text
+
+Base = declarative_base()  # pylint: disable=invalid-name,useless-suppression
+
+
+class Obj(Base):  # pylint: disable=too-few-public-methods
+    """The main (and only) table to store object metadata (hashkey, offset, length, ...)."""
+
+    __tablename__ = "db_object"
+
+    id = Column(Integer, primary_key=True)  # pylint: disable=invalid-name
+
+    # Important: there are parts of the code that rely on the fact that this field is unique.
+    # If you really do not want a uniqueness field, you will need to adapt the code.
+    hashkey = Column(String, nullable=False, unique=True, index=True)
+    compressed = Column(Boolean, nullable=False)
+    size = Column(
+        Integer, nullable=False
+    )  # uncompressed size; if uncompressed, size == length
+    offset = Column(Integer, nullable=False)
+    length = Column(Integer, nullable=False)
+    pack_id = Column(
+        Integer, nullable=False
+    )  # integer ID of the pack in which this entry is stored
+
+
+def get_session(
+    path: str, create: bool = False, raise_if_missing: bool = False
+) -> Optional[Session]:
+    """Return a new session to connect to the pack-index SQLite DB.
+
+    :param create: if True, creates the sqlite file and schema.
+    :param raise_if_missing: ignored if create==True. If create==False, and the index file
+        is missing, either raise an exception (FileNotFoundError) if this flag is True, or return None
+    """
+    if not create and not os.path.exists(path):
+        if raise_if_missing:
+            raise FileNotFoundError("Pack index does not exist")
+        return None
+
+    engine = create_engine(f"sqlite:///{path}", future=True)
+
+    # For the next two bindings, see background on
+    # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl
+    @event.listens_for(engine, "connect")
+    def do_connect(dbapi_connection, _):
+        """Hook function that is called upon connection.
+
+        It modifies the default behavior of SQLite to use WAL and to
+        go back to the 'default' isolation level mode.
+        """
+        # disable pysqlite's emitting of the BEGIN statement entirely.
+        # also stops it from emitting COMMIT before any DDL.
+        dbapi_connection.isolation_level = None
+        # Open the file in WAL mode (see e.g. https://stackoverflow.com/questions/9671490)
+        # This allows to have as many readers as one wants, and a concurrent writer (up to one)
+        # Note that this writes on a journal, on a different packs.idx-wal,
+        # and also creates a packs.idx-shm file.
+        # Note also that when the session is created, you will keep reading from the same version,
+        # so you need to close and reload the session to see the newly written data.
+        # Docs on WAL: https://www.sqlite.org/wal.html
+        cursor = dbapi_connection.cursor()
+        cursor.execute("PRAGMA journal_mode=wal;")
+        cursor.close()
+
+    # For this binding, see background on
+    # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl
+    @event.listens_for(engine, "begin")
+    def do_begin(conn):  # pylint: disable=unused-variable
+        # emit our own BEGIN
+        conn.execute(text("BEGIN"))
+
+    if create:
+        # Create all tables in the engine. This is equivalent to "Create Table"
+        # statements in raw SQL.
+        Base.metadata.create_all(engine)
+
+    # Bind the engine to the metadata of the Base class so that the
+    # declaratives can be accessed through a DBSession instance
+    Base.metadata.bind = engine
+
+    # We set autoflush = False to avoid to lock the DB if just doing queries/reads
+    session = sessionmaker(
+        bind=engine, autoflush=False, autocommit=False, future=True
+    )()
+
+    return session

disk_objectstore/utils.py

@@ -19,14 +19,13 @@
     Iterable,
     Iterator,
     Optional,
+    Sequence,
     Tuple,
     Type,
     Union,
 )
 from zlib import error
 
-from sqlalchemy.engine.result import ResultProxy
-
 from .exceptions import ClosingNotAllowed, ModificationNotAllowed
 
 try:
@@ -1265,7 +1264,7 @@ def detect_where_sorted(  # pylint: disable=too-many-branches, too-many-statemen
         now_left = new_now_left
 
 
-def yield_first_element(iterator: Union[ResultProxy, zip]) -> Iterator[Union[str, int]]:
+def yield_first_element(iterator: Iterable[Sequence]) -> Iterator[Any]:
     """Given an iterator that returns a tuple, return an iterator that yields only the first element of the tuple."""
     for elem in iterator:
         yield elem[0]

performance-benchmarks/validation-calls/performance.txt

@@ -3,7 +3,7 @@ from sqlalchemy import func
 from tqdm import tqdm
 
 
-from disk_objectstore.models import Obj
+from disk_objectstore.database import Obj
 from disk_objectstore import Container
 from disk_objectstore.utils import PackedObjectReader, StreamDecompresser, get_hash
 c = Container('test-newrepo-sdb/')

pyproject.toml

@@ -32,3 +32,4 @@ check_untyped_defs = true
 scripts_are_modules = true
 warn_unused_ignores = true
 warn_redundant_casts = true
+plugins = ["sqlalchemy.ext.mypy.plugin"]

requirements.lock

@@ -20,6 +20,8 @@ distlib==0.3.2
     # via virtualenv
 filelock==3.0.12
     # via virtualenv
+greenlet==1.1.1
+    # via sqlalchemy
 identify==2.2.13
     # via pre-commit
 importlib-metadata==4.7.1
@@ -28,6 +30,7 @@ importlib-metadata==4.7.1
     #   pluggy
     #   pre-commit
     #   pytest
+    #   sqlalchemy
     #   virtualenv
 iniconfig==1.1.1
     # via pytest
@@ -68,7 +71,7 @@ pyyaml==5.4.1
     # via pre-commit
 six==1.16.0
     # via virtualenv
-sqlalchemy==1.3.24
+sqlalchemy==1.4.23
     # via disk-objectstore (setup.py)
 toml==0.10.2
     # via

setup.cfg

@@ -26,7 +26,7 @@ keywords = object store, repository, file store
 [options]
 packages = find:
 install_requires =
-    sqlalchemy<1.4
+    sqlalchemy~=1.4.22
     typing-extensions;python_version < '3.8'
 python_requires = ~=3.7
 include_package_data = True

tests/concurrent_tests/periodic_worker.py

@@ -20,9 +20,10 @@
 
 import click
 import psutil
+from sqlalchemy.sql.expression import select
 
 from disk_objectstore.container import Container, NotExistent, ObjectType
-from disk_objectstore.models import Obj
+from disk_objectstore.database import Obj
 
 MAX_RETRIES_NO_PERM = 1000
 
@@ -348,19 +349,15 @@ def main(
                 session = (
                     container._get_cached_session()  # pylint: disable=protected-access
                 )
-                query = (
-                    session.query(Obj)
-                    .filter(Obj.hashkey == key)
-                    .with_entities(
-                        Obj.pack_id,
-                        Obj.hashkey,
-                        Obj.offset,
-                        Obj.length,
-                        Obj.compressed,
-                        Obj.size,
-                    )
-                )
-                print(list(query))
+                stmt = select(
+                    Obj.pack_id,
+                    Obj.hashkey,
+                    Obj.offset,
+                    Obj.length,
+                    Obj.compressed,
+                    Obj.size,
+                ).where(Obj.hashkey == key)
+                print(list(session.execute(stmt)))
                 raise
 
 

tests/test_container.py

@@ -14,7 +14,7 @@
 import pytest
 
 import disk_objectstore.exceptions as exc
-from disk_objectstore import CompressMode, Container, ObjectType, models, utils
+from disk_objectstore import CompressMode, Container, ObjectType, database, utils
 
 COMPRESSION_ALGORITHMS_TO_TEST = ["zlib+1", "zlib+9"]
 
@@ -2260,9 +2260,9 @@ def test_validate_overlapping_packed(temp_container):  # pylint: disable=invalid
     assert not any(errors.values())
 
     # Change the offset of the second object so that it's overlapping
-    temp_container._get_cached_session().query(models.Obj).filter(
-        models.Obj.hashkey == hashkey_second
-    ).update({models.Obj.offset: models.Obj.offset - 1})
+    temp_container._get_cached_session().query(database.Obj).filter(
+        database.Obj.hashkey == hashkey_second
+    ).update({database.Obj.offset: database.Obj.offset - 1})
 
     errors = temp_container.validate()
     problems = errors.pop("overlapping_packed")

tox.ini

@@ -16,4 +16,6 @@ extras =
     dev
 deps =
     black
+setenv =
+    SQLALCHEMY_WARN_20 = 1
 commands = pytest {posargs}