Use sqlalchemy-helpers

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

pyproject.toml

@@ -21,6 +21,7 @@ packages = [
 
 include = [
     { path = "tests/*", format = "sdist" },
+    { path = "examples/*", format = "sdist" },
 ]
 
 classifiers= [
@@ -40,6 +41,7 @@ arrow = "^1.3.0"
 alembic = "^1.13.1"
 fedora-messaging = "^3.5.0"
 tahrir-messages = "^1.0.2"
+sqlalchemy-helpers = ">=0.13.0"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -50,7 +52,7 @@ pytest = ">=8.1.1"
 pytest-cov = ">=5.0.0"
 
 [tool.poetry.scripts]
-initialize_tahrir_db = "tahrir_api.scripts.initializedb:main"
+sync_tahrir_db = "tahrir_api.scripts.syncdb:main"
 populate_series_in_tahrir_db = "tahrir_api.scripts.populateseries:main"
 
 [build-system]
@@ -83,4 +85,5 @@ exclude_lines = [
 ]
 omit = [
     "tests/*",
+    "tahrir_api/migrations/env.py",
 ]

tahrir_api/dbapi.py

@@ -2,11 +2,12 @@
 #          Remy D <remyd@civx.us>
 # Description: API For interacting with the Tahrir database
 
+import importlib.resources
 from collections import OrderedDict
 from datetime import datetime, timedelta
 
-from sqlalchemy import and_, create_engine, func, not_, text
-from sqlalchemy.orm import scoped_session, sessionmaker
+from sqlalchemy import and_, func, not_, text
+from sqlalchemy_helpers import DatabaseManager
 from tahrir_messages import BadgeAwardV1, PersonLoginFirstV1, PersonRankAdvanceV1
 
 from .model import (
@@ -45,11 +46,15 @@ def __init__(self, dburi=None, session=None, autocommit=True, notification_callb
             raise ValueError("Provide only one, either 'dburi' or 'session'")
 
         self.autocommit = autocommit
-        self.session = session
 
         if dburi:
-            self.session_maker = sessionmaker(bind=create_engine(dburi))
-            self.session = scoped_session(self.session_maker)
+            with importlib.resources.as_file(
+                importlib.resources.files("tahrir_api").joinpath("migrations")
+            ) as alembic_path:
+                self.db_mgr = DatabaseManager(dburi, alembic_path.as_posix())
+            self.session = self.db_mgr.Session()
+        else:
+            self.session = session
 
         self.notification_callback = notification_callback
 
@@ -641,7 +646,7 @@ def add_invitation(self, badge_id, created_on=None, expires_on=None, created_by_
         if self.person_exists(email=created_by_email):
             created_by = self.get_person(created_by_email).id
         else:
-            created_by = "1"
+            created_by = self.session.query(Person).first().id
 
         invitation = Invitation(
             created_on=created_on,

alembic/env.py → tahrir_api/migrations/env.py

@@ -1,7 +1,9 @@
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
+# from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
 
+from alembic import context
 import sqlalchemy as sa
 
 # this is the Alembic Config object, which provides
@@ -10,7 +12,8 @@
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
-fileConfig(config.config_file_name)
+# if config.config_file_name is not None:
+#     fileConfig(config.config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
@@ -27,7 +30,7 @@
 # ... etc.
 
 
-def run_migrations_offline():
+def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
     This configures the context with just a URL
@@ -40,40 +43,37 @@ def run_migrations_offline():
 
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url)
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
 
     with context.begin_transaction():
         context.run_migrations()
 
 
-def run_migrations_online():
+def run_migrations_online() -> None:
     """Run migrations in 'online' mode.
 
     In this scenario we need to create an Engine
     and associate a connection with the context.
 
     """
-    engine = engine_from_config(
-        config.get_section(config.config_ini_section),
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )
 
-    connection = engine.connect()
-    context.configure(connection=connection, target_metadata=target_metadata)
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
 
-    trans = connection.begin()
-    try:
         with context.begin_transaction():
             context.run_migrations()
-            trans.commit()
-    except sa.exc.OperationalError as e:
-        print("SQLite does not allow one of these operations. Rolling back.")
-        print("The exception was:")
-        print(repr(e))
-        trans.rollback()
-    finally:
-        connection.close()
 
 
 if context.is_offline_mode():

tahrir_api/model.py

@@ -7,14 +7,9 @@
 import pygments
 import simplejson
 from sqlalchemy import Column, DateTime, ForeignKey, Unicode, UniqueConstraint
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship, scoped_session, sessionmaker
+from sqlalchemy.orm import object_session, relationship, sessionmaker
 from sqlalchemy.types import Boolean, Integer
-
-DBSession = scoped_session(sessionmaker())
-
-DeclarativeBase = declarative_base()
-DeclarativeBase.query = DBSession.query_property()
+from sqlalchemy_helpers import Base as DeclarativeBase
 
 
 class Issuer(DeclarativeBase):
@@ -27,11 +22,9 @@ class Issuer(DeclarativeBase):
     badges = relationship("Badge", backref="issuer")
     created_on = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
 
-    def __unicode__(self):
+    def __str__(self):
         return str(self.name)
 
-    __str__ = __unicode__
-
     def __json__(self):
         return dict(
             origin=self.origin,
@@ -62,11 +55,9 @@ class Badge(DeclarativeBase):
     created_on = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     tags = Column(Unicode(128))
 
-    def __unicode__(self):
+    def __str__(self):
         return str(self.name)
 
-    __str__ = __unicode__
-
     def __json__(self):
         if self.image.startswith("http"):
             image = self.image
@@ -172,11 +163,9 @@ def gravatar_link(self):
         url = f"http://www.gravatar.com/avatar/{hash}?s={s}&d={d}"
         return url
 
-    def __unicode__(self):
+    def __str__(self):
         return str(self.email)
 
-    __str__ = __unicode__
-
     def __json__(self):
         return dict(
             email=self.email,
@@ -258,11 +247,9 @@ class Assertion(DeclarativeBase):
 
     recipient = Column(Unicode(256), nullable=False, default=recipient_default)
 
-    def __unicode__(self):
+    def __str__(self):
         return str(self.badge) + "<->" + str(self.person)
 
-    __str__ = __unicode__
-
     @property
     def _recipient(self):
         return f"sha256${self.recipient}"
@@ -280,7 +267,8 @@ def __getitem__(self, key):
         return getattr(self, f"__{key}__")()
 
     def __delete__(self):
-        return lambda: DBSession.delete(self)
+        session = object_session(self)
+        return lambda: session.delete(self)
 
     def __pygments__(self):
         html_args = {"full": False}

tahrir_api/scripts/populateseries.py

@@ -2,11 +2,8 @@
 import re
 import sys
 
-import transaction
-from paste.deploy import appconfig
-from sqlalchemy import engine_from_config
-
-from ..model import Badge, DBSession, Milestone, Series
+from ..model import Badge, Milestone, Series
+from .utils import get_db_manager_from_paste
 
 
 def usage(argv):
@@ -15,16 +12,6 @@ def usage(argv):
     sys.exit(1)
 
 
-def _getpathsec(config_uri, name):
-    if "#" in config_uri:
-        path, section = config_uri.split("#", 1)
-    else:
-        path, section = config_uri, "main"
-    if name:
-        section = name
-    return path, section
-
-
 _ROMAN_TO_ARABIC = dict(
     [("I", 1), ("V", 5), ("X", 10), ("L", 50), ("C", 100), ("D", 500), ("M", 1000)]
 )
@@ -81,25 +68,8 @@ def main(argv=sys.argv):
         usage(argv)
 
     config_uri = argv[1]
-    path, section = _getpathsec(config_uri, "pyramid")
-    config_name = "config:%s" % path
-    here_dir = os.getcwd()
-
-    global_conf = None
-    if "OPENSHIFT_APP_NAME" in os.environ:
-        if "OPENSHIFT_MYSQL_DB_URL" in os.environ:
-            template = "{OPENSHIFT_MYSQL_DB_URL}{OPENSHIFT_APP_NAME}"
-        elif "OPENSHIFT_POSTGRESQL_DB_URL" in os.environ:
-            template = "{OPENSHIFT_POSTGRESQL_DB_URL}{OPENSHIFT_APP_NAME}"
-
-        global_conf = {"sqlalchemy.url": template.format(**os.environ)}
-
-    settings = appconfig(config_name, name=section, relative_to=here_dir, global_conf=global_conf)
-
-    engine = engine_from_config(settings, "sqlalchemy.")
-    DBSession.configure(bind=engine)
-
-    with DBSession() as session:
+    db_mgr = get_db_manager_from_paste(config_uri)
+    with db_mgr.Session() as session:
         for badge in session.query(Badge).all():
             if badge.milestone:
                 # Skip badges that already are in some series.

tahrir_api/scripts/syncdb.py

@@ -0,0 +1,19 @@
+import os
+import sys
+
+from .utils import get_db_manager_from_paste
+
+
+def usage(argv):
+    cmd = os.path.basename(argv[0])
+    print(f"usage: {cmd} <config_uri>\n '(example: \"{cmd} development.ini\"'")
+    sys.exit(1)
+
+
+def main(argv=sys.argv):
+    if len(argv) != 2:
+        usage(argv)
+
+    config_uri = argv[1]
+    db_mgr = get_db_manager_from_paste(config_uri)
+    db_mgr.sync()

tahrir_api/scripts/initializedb.py → tahrir_api/scripts/utils.py

@@ -1,19 +1,10 @@
-import datetime
+import importlib.resources
 import os
-import pprint
-import sys
 
-import transaction
 from paste.deploy import appconfig
-from sqlalchemy import engine_from_config
+from sqlalchemy_helpers import DatabaseManager
 
-from ..model import Assertion, Badge, DBSession, DeclarativeBase, Issuer, Person
-
-
-def usage(argv):
-    cmd = os.path.basename(argv[0])
-    print(f"usage: {cmd} <config_uri>\n '(example: \"{cmd} development.ini\"'")
-    sys.exit(1)
+from ..model import Assertion, Badge, DeclarativeBase, Issuer, Person  # noqa: F401
 
 
 def _getpathsec(config_uri, name):
@@ -26,11 +17,7 @@ def _getpathsec(config_uri, name):
     return path, section
 
 
-def main(argv=sys.argv):
-    if len(argv) != 2:
-        usage(argv)
-
-    config_uri = argv[1]
+def get_db_manager_from_paste(config_uri):
     path, section = _getpathsec(config_uri, "pyramid")
     config_name = "config:%s" % path
     here_dir = os.getcwd()
@@ -46,6 +33,10 @@ def main(argv=sys.argv):
 
     settings = appconfig(config_name, name=section, relative_to=here_dir, global_conf=global_conf)
 
-    engine = engine_from_config(settings, "sqlalchemy.")
-    DBSession.configure(bind=engine)
-    DeclarativeBase.metadata.create_all(engine)
+    prefix = "sqlalchemy."
+    db_options = {key[len(prefix) :]: settings[key] for key in settings if key.startswith(prefix)}
+    dburi = db_options.pop("url")
+    with importlib.resources.as_file(
+        importlib.resources.files("tahrir_api").joinpath("migrations")
+    ) as alembic_path:
+        return DatabaseManager(dburi, alembic_path.as_posix())