Use SQLAlchemy for database backend

We're using it fairly minimally: essentially still writing manual
queries, but now using SQLAlchemy to handle placeholder binding which is
nicer, and portable.

The API now goes like this:

- `db.get_conn()` returns a connection from the connection pool;
  web.appdb, in particular, is a preloaded, app-context connection
  object that most code uses.

- `web.query()` does an SQLAlchemy 'text' query, which is almost like
  regular prepare+bind+execute, except that it always uses `:name` for
  placeholders and binds using `name=...` keyword arguments.  That is,
  instead of:

      db.execute("SELECT token FROM rooms WHERE id = ?", (123,))

  we now do:

      query("SELECT token FROM rooms WHERE id = :id", id=123)

  or equivalently:

      db.query(appdb, "SELECT token FROM rooms WHERE id = :id", id=123)

  (the latter version is more useful where an app context doesn't exist
  and you need to pass in some other conn, such as is now done in
  __main__).

- transactions are now controlled with a `with appdb.begin_nested():`
  context, replacing the custom nested tx handler I made for sqlite.

We *could* start using more advanced SQLAlchemy query composition, but
that seems like more of a pain rather than something useful.  (I *don't*
want to use SQLAlchemy ORM, because in my experience ORM just gets in
the way as soon as you want non-trivial database structure, which we
have here).

This will allow us to (easily) add postgresql support.  (Other database
are probably a no-go: SQLite has long looked up to postgresql for
features and syntax and so the two are very similar dialects).

sogs/__main__.py

@@ -1,7 +1,7 @@
 from argparse import ArgumentParser as AP, RawDescriptionHelpFormatter
 import re
 import sys
-import sqlite3
+import sqlalchemy
 
 from . import db
 from . import config
@@ -139,17 +139,20 @@ def print_room(room: model.Room):
         )
         sys.exit(1)
 
-    try:
-        with db.tx() as cur:
-            cur.execute(
-                "INSERT INTO rooms(token, name, description) VALUES(?, ?, ?)",
-                [args.add_room, args.name or args.add_room, args.description],
+    with db.get_conn() as conn:
+        try:
+            db.query(
+                conn,
+                "INSERT INTO rooms(token, name, description) VALUES(:t, :n, :d)",
+                t=args.add_room,
+                n=args.name or args.add_room,
+                d=args.description,
             )
-    except sqlite3.IntegrityError:
-        print(f"Error: room '{args.add_room}' already exists!", file=sys.stderr)
-        sys.exit(1)
-    print(f"Created room {args.add_room}:")
-    print_room(model.Room(token=args.add_room))
+        except sqlalchemy.exc.IntegrityError:
+            print(f"Error: room '{args.add_room}' already exists!", file=sys.stderr)
+            sys.exit(1)
+        print(f"Created room {args.add_room}:")
+        print_room(model.Room(token=args.add_room))
 
 elif args.delete_room:
     try:
@@ -164,9 +167,9 @@ def print_room(room: model.Room):
     else:
         res = input("Are you sure you want to delete this room? [yN] ")
     if res.startswith("y") or res.startswith("Y"):
-        with db.tx() as cur:
-            cur.execute("DELETE FROM rooms WHERE token = ?", [args.delete_room])
-            count = cur.rowcount
+        with db.get_conn() as conn:
+            result = db.query(conn, "DELETE FROM rooms WHERE token = ?", [args.delete_room])
+            count = result.rowcount
         print("Room deleted.")
     else:
         print("Aborted.")

sogs/cleanup.py

@@ -1,9 +1,7 @@
-import logging
 import os
 import time
 
-from .web import app
-from . import db
+from .web import app, appdb, query
 from . import config
 
 # Cleanup interval, in seconds.
@@ -28,16 +26,15 @@ def cleanup():
 
 
 def prune_files():
-    with db.tx() as cur:
+    with appdb.begin_nested():
         # Would love to use a single DELETE ... RETURNING here, but that requires sqlite 3.35+.
         now = time.time()
-        cur.execute("SELECT path FROM files WHERE expiry < ?", (now,))
-        to_remove = [row[0] for row in cur]
+        to_remove = [row[0] for row in query("SELECT path FROM files WHERE expiry < :exp", exp=now)]
 
         if not to_remove:
             return 0
 
-        cur.execute("DELETE FROM files WHERE expiry <= ?", (now,))
+        query("DELETE FROM files WHERE expiry < :exp", exp=now)
 
     # Committed the transaction, so the files are gone: now go ahead and remove them from disk.
     unlink_count = 0
@@ -60,52 +57,49 @@ def prune_files():
 
 
 def prune_message_history():
-    with db.tx() as cur:
-        cur.execute(
-            "DELETE FROM message_history WHERE replaced < ?",
-            (time.time() - config.MESSAGE_HISTORY_PRUNE_THRESHOLD * 86400,),
-        )
-        count = cur.rowcount
+    count = query(
+        "DELETE FROM message_history WHERE replaced < :t",
+        t=time.time() - config.MESSAGE_HISTORY_PRUNE_THRESHOLD * 86400,
+    ).rowcount
 
     if count > 0:
         app.logger.info("Pruned {} message edit/deletion records".format(count))
     return count
 
 
 def prune_room_activity():
-    with db.tx() as cur:
-        cur.execute(
-            "DELETE FROM room_users WHERE last_active < ?",
-            (time.time() - config.ROOM_ACTIVE_PRUNE_THRESHOLD * 86400,),
-        )
-        count = cur.rowcount
+    with appdb.begin_nested():
+        count = query(
+            "DELETE FROM room_users WHERE last_active < :t",
+            t=time.time() - config.ROOM_ACTIVE_PRUNE_THRESHOLD * 86400,
+        ).rowcount
 
     if count > 0:
         app.logger.info("Prune {} old room activity records".format(count))
     return count
 
 
 def apply_permission_updates():
-    with db.tx() as cur:
+    with appdb.begin_nested():
         now = time.time()
-        cur.execute(
+        num_applied = query(
             """
             INSERT INTO user_permission_overrides (room, user, read, write, upload, banned)
             SELECT room, user, read, write, upload, banned FROM user_permission_futures
-                WHERE at <= ?
+                WHERE at <= :now
             ON CONFLICT (room, user) DO UPDATE SET
                 read = COALESCE(excluded.read, read),
                 write = COALESCE(excluded.write, write),
                 upload = COALESCE(excluded.upload, upload),
                 banned = COALESCE(excluded.banned, banned)
             """,
-            (now,),
-        )
-        num_applied = cur.rowcount
+            now=now,
+        ).rowcount
         if not num_applied:
             return 0
 
-        cur.execute("DELETE FROM user_permission_futures WHERE at <= ?", (now,))
+        query("DELETE FROM user_permission_futures WHERE at <= :now", now=now)
 
-    logging.info("Applied {} scheduled user permission updates".format(num_applied))
+    if num_applied > 0:
+        app.logger.info("Applied {} scheduled user permission updates".format(num_applied))
     return num_applied

sogs/config.py

@@ -10,7 +10,7 @@
 coloredlogs.install(milliseconds=True, isatty=True, logger=logger)
 
 # Default config settings; most of these are configurable via config.ini (see it for details).
-DB_PATH = 'sogs.db'
+DB_URL = 'sqlite:///sogs.db'
 DB_SCHEMA_FILE = 'schema.sql'  # Not configurable, just a constant
 KEY_FILE = 'key_x25519'
 URL_BASE = 'http://example.net'
@@ -71,7 +71,9 @@ def load_config():
     # test lambda returns True/False for validation (if None/omitted, accept anything)
     # value lambda extracts the value (if None/omitted use str value as-is)
     setting_map = {
-        'db': {'url': ('DB_PATH', lambda x: x.startswith('sqlite:///'), lambda x: x[10:])},
+        'db': {
+            'url': ('DB_URL', lambda x: x.startswith('sqlite:///'))
+        },
         'crypto': {'key_file': ('KEY_FILE',)},
         'net': {
             'base_url': ('URL_BASE', lambda x: re.search('^https?://.', x)),