prs-fleshgolem-2070: feat: sqlalchemy 2.0 (#2096)

* upgrade sqlalchemy to 2.0

* rewrite all db models to sqla 2.0 mapping api

* fix some importing and typing weirdness

* fix types of a lot of nullable columns

* remove get_ref methods

* fix issues found by tests

* rewrite all queries in repository_recipe to 2.0 style

* rewrite all repository queries to 2.0 api

* rewrite all remaining queries to 2.0 api

* remove now-unneeded __allow_unmapped__ flag

* remove and fix some unneeded cases of "# type: ignore"

* fix formatting

* bump black version

* run black

* can this please be the last one. okay. just. okay.

* fix repository errors

* remove return

* drop open API validator

---------

Co-authored-by: Sören Busch <fleshgolem@gmx.net>

.gitignore

@@ -159,3 +159,5 @@ dev/code-generation/generated/test_routes.py
 mealie/services/parser_services/crfpp/model.crfmodel
 lcov.info
 dev/code-generation/openapi.json
+
+.run/

.pre-commit-config.yaml

@@ -11,10 +11,6 @@ repos:
       - id: trailing-whitespace
         exclude: ^tests/data/
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
       - id: black
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.241
-    hooks:
-      - id: ruff

dev/code-generation/gen_py_pytest_routes.py

@@ -25,7 +25,6 @@ def get_path_objects(app: FastAPI):
     for key, value in app.openapi().items():
         if key == "paths":
             for key, value in value.items():
-
                 paths.append(
                     PathObject(
                         route_object=RouteObject(key),
@@ -50,7 +49,6 @@ def read_template(file: Path):
 
 
 def generate_python_templates(static_paths: list[PathObject], function_paths: list[PathObject]):
-
     template = Template(read_template(CodeTemplates.pytest_routes))
     content = template.render(
         paths={

dev/code-generation/gen_py_schema_exports.py

@@ -79,14 +79,12 @@ def find_modules(root: pathlib.Path) -> list[Modules]:
     modules: list[Modules] = []
     for file in root.iterdir():
         if file.is_dir() and file.name not in SKIP:
-
             modules.append(Modules(directory=file))
 
     return modules
 
 
 def main():
-
     modules = find_modules(SCHEMA_PATH)
 
     for module in modules:

dev/scripts/all_recipes_stress_test.py

@@ -232,7 +232,6 @@ def recipe_data(name: str, slug: str, id: str, userId: str, groupId: str) -> dic
 
 
 def login(username="changeme@email.com", password="MyPassword"):
-
     payload = {"username": username, "password": password}
     r = requests.post("http://localhost:9000/api/auth/token", payload)
 

gunicorn_conf.py

@@ -9,7 +9,6 @@ class GunicornConfig:
     """Configuration to generate the properties for Gunicorn"""
 
     def __init__(self):
-
         # Env Variables
         self.host = os.getenv("HOST", "127.0.0.1")
         self.port = os.getenv("API_PORT", "9000")

mealie/db/db_setup.py

@@ -15,14 +15,9 @@ def sql_global_init(db_url: str):
     if "sqlite" in db_url:
         connect_args["check_same_thread"] = False
 
-    engine = sa.create_engine(
-        db_url,
-        echo=False,
-        connect_args=connect_args,
-        pool_pre_ping=True,
-    )
-
-    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    engine = sa.create_engine(db_url, echo=False, connect_args=connect_args, pool_pre_ping=True, future=True)
+
+    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine, future=True)
 
     return SessionLocal, engine
 

mealie/db/init_db.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from time import sleep
 
-from sqlalchemy import engine, orm
+from sqlalchemy import engine, orm, text
 
 from alembic import command, config, script
 from alembic.config import Config
@@ -59,7 +59,7 @@ def safe_try(func: Callable):
 
 def connect(session: orm.Session) -> bool:
     try:
-        session.execute("SELECT 1")
+        session.execute(text("SELECT 1"))
         return True
     except Exception as e:
         logger.error(f"Error connecting to database: {e}")

mealie/db/models/_all_models.py

@@ -1,5 +1,5 @@
 from .group import *
 from .labels import *
-from .recipe.recipe import *  # type: ignore
+from .recipe import *
 from .server import *
 from .users import *

mealie/db/models/_model_base.py

@@ -1,37 +1,19 @@
 from datetime import datetime
 
-from sqlalchemy import Column, DateTime, Integer
-from sqlalchemy.ext.declarative import as_declarative
-from sqlalchemy.orm import declarative_base
-from sqlalchemy.orm.session import Session
+from sqlalchemy import DateTime, Integer
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 
-@as_declarative()
-class Base:
-    id = Column(Integer, primary_key=True)
-    created_at = Column(DateTime, default=datetime.now)
-    update_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+class SqlAlchemyBase(DeclarativeBase):
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    created_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now)
+    update_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)
 
 
 class BaseMixins:
     """
     `self.update` method which directly passing arguments to the `__init__`
-    `cls.get_ref` method which will return the object from the database or none. Useful for many-to-many relationships.
     """
 
     def update(self, *args, **kwarg):
         self.__init__(*args, **kwarg)
-
-    @classmethod
-    def get_ref(cls, match_value: str, match_attr: str | None = None, session: Session | None = None):
-        match_attr = match_attr or cls.Config.get_attr  # type: ignore
-
-        if match_value is None or session is None:
-            return None
-
-        eff_ref = getattr(cls, match_attr)
-
-        return session.query(cls).filter(eff_ref == match_value).one_or_none()
-
-
-SqlAlchemyBase = declarative_base(cls=Base, constructor=None)

mealie/db/models/_model_utils/auto_init.py

@@ -2,13 +2,13 @@
 from uuid import UUID
 
 from pydantic import BaseModel, Field, NoneStr
+from sqlalchemy import select
 from sqlalchemy.orm import MANYTOMANY, MANYTOONE, ONETOMANY, Session
-from sqlalchemy.orm.decl_api import DeclarativeMeta
 from sqlalchemy.orm.mapper import Mapper
 from sqlalchemy.orm.relationships import RelationshipProperty
 from sqlalchemy.sql.base import ColumnCollection
-from sqlalchemy.util._collections import ImmutableProperties
 
+from .._model_base import SqlAlchemyBase
 from .helpers import safe_call
 
 
@@ -26,7 +26,7 @@ class AutoInitConfig(BaseModel):
     # auto_create: bool = False
 
 
-def _get_config(relation_cls: DeclarativeMeta) -> AutoInitConfig:
+def _get_config(relation_cls: type[SqlAlchemyBase]) -> AutoInitConfig:
     """
     Returns the config for the given class.
     """
@@ -45,7 +45,7 @@ def _get_config(relation_cls: DeclarativeMeta) -> AutoInitConfig:
     return cfg
 
 
-def get_lookup_attr(relation_cls: DeclarativeMeta) -> str:
+def get_lookup_attr(relation_cls: type[SqlAlchemyBase]) -> str:
     """Returns the primary key attribute of the related class as a string.
 
     Args:
@@ -73,24 +73,25 @@ def handle_many_to_many(session, get_attr, relation_cls, all_elements: list[dict
     return handle_one_to_many_list(session, get_attr, relation_cls, all_elements)
 
 
-def handle_one_to_many_list(session: Session, get_attr, relation_cls, all_elements: list[dict] | list[str]):
+def handle_one_to_many_list(
+    session: Session, get_attr, relation_cls: type[SqlAlchemyBase], all_elements: list[dict] | list[str]
+):
     elems_to_create: list[dict] = []
     updated_elems: list[dict] = []
 
     cfg = _get_config(relation_cls)
 
     for elem in all_elements:
         elem_id = elem.get(get_attr, None) if isinstance(elem, dict) else elem
-        existing_elem = session.query(relation_cls).filter_by(**{get_attr: elem_id}).one_or_none()
+        stmt = select(relation_cls).filter_by(**{get_attr: elem_id})
+        existing_elem = session.execute(stmt).scalars().one_or_none()
 
-        is_dict = isinstance(elem, dict)
-
-        if existing_elem is None and is_dict:
-            elems_to_create.append(elem)  # type: ignore
+        if existing_elem is None and isinstance(elem, dict):
+            elems_to_create.append(elem)
             continue
 
-        elif is_dict:
-            for key, value in elem.items():  # type: ignore
+        elif isinstance(elem, dict):
+            for key, value in elem.items():
                 if key not in cfg.exclude:
                     setattr(existing_elem, key, value)
 
@@ -110,7 +111,7 @@ def auto_init():  # sourcery no-metrics
 
     def decorator(init):
         @wraps(init)
-        def wrapper(self: DeclarativeMeta, *args, **kwargs):  # sourcery no-metrics
+        def wrapper(self: SqlAlchemyBase, *args, **kwargs):  # sourcery no-metrics
             """
             Custom initializer that allows nested children initialization.
             Only keys that are present as instance's class attributes are allowed.
@@ -120,14 +121,14 @@ def wrapper(self: DeclarativeMeta, *args, **kwargs):  # sourcery no-metrics
             Ref: https://github.com/tiangolo/fastapi/issues/2194
             """
             cls = self.__class__
-
-            exclude = _get_config(cls).exclude
+            config = _get_config(cls)
+            exclude = config.exclude
 
             alchemy_mapper: Mapper = self.__mapper__
             model_columns: ColumnCollection = alchemy_mapper.columns
-            relationships: ImmutableProperties = alchemy_mapper.relationships
+            relationships = alchemy_mapper.relationships
 
-            session = kwargs.get("session", None)
+            session: Session = kwargs.get("session", None)
 
             if session is None:
                 raise ValueError("Session is required to initialize the model with `auto_init`")
@@ -151,7 +152,7 @@ def wrapper(self: DeclarativeMeta, *args, **kwargs):  # sourcery no-metrics
                     relation_dir = prop.direction
 
                     # Identifies the parent class of the related object.
-                    relation_cls: DeclarativeMeta = prop.mapper.entity
+                    relation_cls: type[SqlAlchemyBase] = prop.mapper.entity
 
                     # Identifies if the relationship was declared with use_list=True
                     use_list: bool = prop.uselist
@@ -174,7 +175,8 @@ def wrapper(self: DeclarativeMeta, *args, **kwargs):  # sourcery no-metrics
                                 raise ValueError(f"Expected 'id' to be provided for {key}")
 
                         if isinstance(val, (str, int, UUID)):
-                            instance = session.query(relation_cls).filter_by(**{get_attr: val}).one_or_none()
+                            stmt = select(relation_cls).filter_by(**{get_attr: val})
+                            instance = session.execute(stmt).scalars().one_or_none()
                             setattr(self, key, instance)
                         else:
                             # If the value is not of the type defined above we assume that it isn't a valid id

mealie/db/models/group/cookbook.py

@@ -1,33 +1,41 @@
-from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, orm
+from typing import TYPE_CHECKING, Optional
+
+from sqlalchemy import Boolean, ForeignKey, Integer, String, orm
+from sqlalchemy.orm import Mapped, mapped_column
 
 from .._model_base import BaseMixins, SqlAlchemyBase
 from .._model_utils import auto_init, guid
 from ..recipe.category import Category, cookbooks_to_categories
 from ..recipe.tag import Tag, cookbooks_to_tags
 from ..recipe.tool import Tool, cookbooks_to_tools
 
+if TYPE_CHECKING:
+    from group import Group
+
 
 class CookBook(SqlAlchemyBase, BaseMixins):
     __tablename__ = "cookbooks"
-    id = Column(guid.GUID, primary_key=True, default=guid.GUID.generate)
-    position = Column(Integer, nullable=False, default=1)
+    id: Mapped[guid.GUID] = mapped_column(guid.GUID, primary_key=True, default=guid.GUID.generate)
+    position: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
 
-    group_id = Column(guid.GUID, ForeignKey("groups.id"))
-    group = orm.relationship("Group", back_populates="cookbooks")
+    group_id: Mapped[guid.GUID | None] = mapped_column(guid.GUID, ForeignKey("groups.id"))
+    group: Mapped[Optional["Group"]] = orm.relationship("Group", back_populates="cookbooks")
 
-    name = Column(String, nullable=False)
-    slug = Column(String, nullable=False)
-    description = Column(String, default="")
-    public = Column(Boolean, default=False)
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    slug: Mapped[str] = mapped_column(String, nullable=False)
+    description: Mapped[str | None] = mapped_column(String, default="")
+    public: Mapped[str | None] = mapped_column(Boolean, default=False)
 
-    categories = orm.relationship(Category, secondary=cookbooks_to_categories, single_parent=True)
-    require_all_categories = Column(Boolean, default=True)
+    categories: Mapped[list[Category]] = orm.relationship(
+        Category, secondary=cookbooks_to_categories, single_parent=True
+    )
+    require_all_categories: Mapped[bool | None] = mapped_column(Boolean, default=True)
 
-    tags = orm.relationship(Tag, secondary=cookbooks_to_tags, single_parent=True)
-    require_all_tags = Column(Boolean, default=True)
+    tags: Mapped[list[Tag]] = orm.relationship(Tag, secondary=cookbooks_to_tags, single_parent=True)
+    require_all_tags: Mapped[bool | None] = mapped_column(Boolean, default=True)
 
-    tools = orm.relationship(Tool, secondary=cookbooks_to_tools, single_parent=True)
-    require_all_tools = Column(Boolean, default=True)
+    tools: Mapped[list[Tool]] = orm.relationship(Tool, secondary=cookbooks_to_tools, single_parent=True)
+    require_all_tools: Mapped[bool | None] = mapped_column(Boolean, default=True)
 
     @auto_init()
     def __init__(self, **_) -> None: