Upgrade psql client (#100)

* build: Upgrade psycopg, v2 => v3

* build: Specify dialect in database uri

* fix: Update sql for psycopg v3 semantics

* fix: Update sql for psycopg v3 semantics

mmmaybe?

* build: Specify dialect in workflow db uri

.github/workflows/checks.yml

@@ -13,7 +13,7 @@ env:
   COLANDR_DB_USER: "colandr_app"
   COLANDR_DB_PASSWORD: "password"
   COLANDR_DB_NAME: "colandr"
-  COLANDR_DATABASE_URI: "postgresql://colandr_app:password@localhost:5432/colandr"
+  COLANDR_DATABASE_URI: "postgresql+psycopg://colandr_app:password@localhost:5432/colandr"
   COLANDR_SECRET_KEY: "colandr_secret_key"
   COLANDR_APP_DIR: "/tmp"
   COLANDR_MAIL_USERNAME: "colandr_mail_username"

colandr/apis/resources/citation_screenings.py

@@ -454,7 +454,8 @@ def post(self, args, review_id, user_id):
         status_counts_stmt = (
             sa.select(Study.citation_status, db.func.count(1))
             .filter_by(review_id=review_id, dedupe_status="not_duplicate")
-            .filter(Study.citation_status.in_(["included", "excluded"]))
+            # .filter(Study.citation_status.in_(["included", "excluded"]))
+            .filter(Study.citation_status == sa.any_(["included", "excluded"]))
             .group_by(Study.citation_status)
         )
         status_counts: dict[str, int] = {

colandr/apis/resources/fulltext_screenings.py

@@ -453,7 +453,8 @@ def post(self, args, review_id, user_id):
         status_counts_stmt = (
             sa.select(Study.fulltext_status, db.func.count(1))
             .filter_by(review_id=review_id)
-            .filter(Study.fulltext_status.in_(["included", "excluded"]))
+            # .filter(Study.fulltext_status.in_(["included", "excluded"]))
+            .filter(Study.fulltext_status == sa.any_(["included", "excluded"]))
             .group_by(Study.fulltext_status)
         )
         status_counts: dict[str, int] = {

colandr/apis/resources/reviews.py

@@ -187,7 +187,9 @@ def get(self, fields, _review_ids):
         current_user = jwtext.get_current_user()
         if current_user.is_admin is True and _review_ids is not None:
             reviews = (
-                db.session.execute(sa.select(Review).filter(Review.id.in_(_review_ids)))
+                db.session.execute(
+                    sa.select(Review).filter(Review.id == sa.any_(_review_ids))
+                )
                 .scalars()
                 .all()
             )

colandr/tasks.py

@@ -133,7 +133,8 @@ def deduplicate_citations(review_id: int):
     stmt = (
         sa.select(Study.id)
         .where(Study.review_id == review_id)
-        .where(Study.citation_status.in_(["included", "excluded"]))
+        # .where(Study.citation_status.in_(["included", "excluded"]))
+        .where(Study.citation_status == sa.any_(["included", "excluded"]))
     )
     incl_excl_cids = set(db.session.execute(stmt).scalars().all())
 
@@ -171,7 +172,8 @@ def deduplicate_citations(review_id: int):
                     ).label("n_null_cols"),
                 )
                 .where(Citation.review_id == review_id)
-                .where(Citation.id.in_(int_cids))
+                # .where(Citation.id.in_(int_cids))
+                .where(Citation.id == sa.any_(int_cids))
                 .order_by(sa.text("n_null_cols ASC"))
                 .limit(1)
             )
@@ -395,7 +397,8 @@ def train_citation_ranking_model(review_id: int):
         .where(Study.id == Citation.id)
         .where(Study.review_id == review_id)
         .where(Study.dedupe_status == "not_duplicate")
-        .where(Study.citation_status.in_(["included", "excluded"]))
+        # .where(Study.citation_status.in_(["included", "excluded"]))
+        .where(Study.citation_status == sa.any_(["included", "excluded"]))
         .where(Citation.text_content_vector_rep != [])
     )
     results = db.session.execute(stmt)

compose.yml

@@ -68,7 +68,7 @@ services:
     env_file: ".env"
     environment:
       - FLASK_APP=colandr.app:create_app()
-      - COLANDR_DATABASE_URI=postgresql://${COLANDR_DB_USER}:${COLANDR_DB_PASSWORD}@colandr-db:5432/${COLANDR_DB_NAME}
+      - COLANDR_DATABASE_URI=postgresql+psycopg://${COLANDR_DB_USER}:${COLANDR_DB_PASSWORD}@colandr-db:5432/${COLANDR_DB_NAME}
       - COLANDR_CELERY_BROKER_URL=redis://colandr-broker:6379/0
       - COLANDR_CELERY_RESULT_BACKEND=redis://colandr-broker:6379/0
       - COLANDR_REDIS_HOST=colandr-broker
@@ -92,7 +92,7 @@ services:
     stop_signal: SIGINT
     env_file: ".env"
     environment:
-      - COLANDR_DATABASE_URI=postgresql://${COLANDR_DB_USER}:${COLANDR_DB_PASSWORD}@colandr-db:5432/${COLANDR_DB_NAME}
+      - COLANDR_DATABASE_URI=postgresql+psycopg://${COLANDR_DB_USER}:${COLANDR_DB_PASSWORD}@colandr-db:5432/${COLANDR_DB_NAME}
       - COLANDR_CELERY_BROKER_URL=redis://colandr-broker:6379/0
       - COLANDR_CELERY_RESULT_BACKEND=redis://colandr-broker:6379/0
       - COLANDR_REDIS_HOST=colandr-broker

pyproject.toml

@@ -25,9 +25,9 @@ dependencies = [
   "flask~=3.0",
   "flask-caching~=2.1.0",
   "flask-jwt-extended~=4.5.0",
-  "flask_mail>=0.9.1",         # TODO: this package is dead, we should replace it
+  "flask_mail>=0.9.1",           # TODO: this package is dead, we should replace it
   "flask_migrate~=4.0.0",
-  "flask_sqlalchemy~=3.1.0",   # constrained by sqlalchemy
+  "flask_sqlalchemy~=3.1.0",     # constrained by sqlalchemy
   "flask-restx~=1.3.0",
   "ftfy~=6.0",
   "gunicorn~=21.0",
@@ -36,7 +36,7 @@ dependencies = [
   "markupsafe~=2.0",
   "marshmallow~=3.19.0",
   "numpy~=1.20",
-  "psycopg2-binary~=2.9.0",
+  "psycopg[binary,pool]~=3.1.0",
   "pymupdf~=1.22.0",
   "python-dateutil~=2.8",
   "python-dotenv~=1.0.0",

requirements/prod.txt

@@ -20,7 +20,7 @@ joblib~=1.0
 markupsafe~=2.0
 marshmallow~=3.19.0
 numpy~=1.20
-psycopg2-binary~=2.9.0
+psycopg[binary,pool]~=3.1.0
 pymupdf~=1.22.0
 python-dateutil~=2.8
 python-dotenv~=1.0.0