TN-3292 fix performance of patient list

portal/config/eproms/ScheduledJob.json

@@ -276,6 +276,14 @@
       "schedule": "0 0 0 0 0",
       "task": "raise_background_exception_task"
     },
+    {
+      "active": false,
+      "args": null,
+      "name": "Populate Patient List",
+      "resourceType": "ScheduledJob",
+      "schedule": "0 0 0 0 0",
+      "task": "cache_patient_list"
+    },
     {
       "active": true,
       "args": null,

portal/migrations/versions/038a1a5f4218_.py

@@ -0,0 +1,76 @@
+"""add patient_list table for paginated /patients view
+
+Revision ID: 038a1a5f4218
+Revises: daee63f50d35
+Create Date: 2024-09-30 16:10:26.216512
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '038a1a5f4218'
+down_revision = 'daee63f50d35'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('patient_list',
+    sa.Column('userid', sa.Integer(), nullable=False),
+    sa.Column('study_id', sa.Text(), nullable=True),
+    sa.Column('firstname', sa.String(length=64), nullable=True),
+    sa.Column('lastname', sa.String(length=64), nullable=True),
+    sa.Column('birthdate', sa.Date(), nullable=True),
+    sa.Column('email', sa.String(length=120), nullable=True),
+    sa.Column('questionnaire_status', sa.Text(), nullable=True),
+    sa.Column('empro_status', sa.Text(), nullable=True),
+    sa.Column('clinician', sa.Text(), nullable=True),
+    sa.Column('action_state', sa.Text(), nullable=True),
+    sa.Column('visit', sa.Text(), nullable=True),
+    sa.Column('empro_visit', sa.Text(), nullable=True),
+    sa.Column('consentdate', sa.DateTime(), nullable=True),
+    sa.Column('empro_consentdate', sa.DateTime(), nullable=True),
+    sa.Column('org_name', sa.Text(), nullable=True),
+    sa.Column('deleted', sa.Boolean(), nullable=True),
+    sa.Column('test_role', sa.Boolean(), nullable=True),
+    sa.Column('org_id', sa.Integer(), nullable=True),
+    sa.Column('last_updated', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['org_id'], ['organizations.id'], ),
+    sa.PrimaryKeyConstraint('userid')
+    )
+    op.create_index(op.f('ix_patient_list_action_state'), 'patient_list', ['action_state'], unique=False)
+    op.create_index(op.f('ix_patient_list_birthdate'), 'patient_list', ['birthdate'], unique=False)
+    op.create_index(op.f('ix_patient_list_clinician'), 'patient_list', ['clinician'], unique=False)
+    op.create_index(op.f('ix_patient_list_consentdate'), 'patient_list', ['consentdate'], unique=False)
+    op.create_index(op.f('ix_patient_list_email'), 'patient_list', ['email'], unique=False)
+    op.create_index(op.f('ix_patient_list_empro_consentdate'), 'patient_list', ['empro_consentdate'], unique=False)
+    op.create_index(op.f('ix_patient_list_empro_status'), 'patient_list', ['empro_status'], unique=False)
+    op.create_index(op.f('ix_patient_list_empro_visit'), 'patient_list', ['empro_visit'], unique=False)
+    op.create_index(op.f('ix_patient_list_firstname'), 'patient_list', ['firstname'], unique=False)
+    op.create_index(op.f('ix_patient_list_lastname'), 'patient_list', ['lastname'], unique=False)
+    op.create_index(op.f('ix_patient_list_org_name'), 'patient_list', ['org_name'], unique=False)
+    op.create_index(op.f('ix_patient_list_questionnaire_status'), 'patient_list', ['questionnaire_status'], unique=False)
+    op.create_index(op.f('ix_patient_list_study_id'), 'patient_list', ['study_id'], unique=False)
+    op.create_index(op.f('ix_patient_list_visit'), 'patient_list', ['visit'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_patient_list_visit'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_study_id'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_questionnaire_status'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_org_name'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_lastname'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_firstname'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_empro_visit'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_empro_status'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_empro_consentdate'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_email'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_consentdate'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_clinician'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_birthdate'), table_name='patient_list')
+    op.drop_index(op.f('ix_patient_list_action_state'), table_name='patient_list')
+    op.drop_table('patient_list')
+    # ### end Alembic commands ###

portal/migrations/versions/5a300be640fb_.py

@@ -0,0 +1,52 @@
+"""empty message
+
+Revision ID: 5a300be640fb
+Revises: 038a1a5f4218
+Create Date: 2024-10-08 14:34:28.085963
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '5a300be640fb'
+down_revision = '038a1a5f4218'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('adherence_data_patient_id_fkey', 'adherence_data')
+    op.create_foreign_key(
+        'adherence_data_patient_id_fkey',
+        'adherence_data',
+        'users', ['patient_id'], ['id'], ondelete='cascade')
+    op.alter_column('patient_list', 'last_updated',
+               existing_type=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.create_foreign_key(
+        'patient_list_userid_fkey',
+        'patient_list',
+        'users', ['userid'], ['id'], ondelete='cascade')
+    op.drop_constraint('research_data_subject_id_fkey', 'research_data', type_='foreignkey')
+    op.create_foreign_key(
+        'research_data_subject_id_fkey',
+        'research_data',
+        'users', ['subject_id'], ['id'], ondelete='cascade')
+    op.drop_constraint('research_data_questionnaire_response_id_fkey', 'research_data', type_='foreignkey')
+    op.create_foreign_key(
+        'research_data_questionnaire_response_id_fkey',
+        'research_data',
+        'questionnaire_responses', ['questionnaire_response_id'], ['id'], ondelete='cascade')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('research_data_subject_id_fkey', 'research_data', type_='foreignkey')
+    op.create_foreign_key('research_data_subject_id_fkey', 'research_data', 'users', ['subject_id'], ['id'])
+    op.drop_constraint('patient_list_userid_fkey', 'patient_list', type_='foreignkey')
+    op.alter_column('patient_list', 'last_updated',
+               existing_type=postgresql.TIMESTAMP(),
+               nullable=False)
+    # ### end Alembic commands ###

portal/models/adherence_data.py

@@ -23,7 +23,8 @@ class AdherenceData(db.Model):
     """
     __tablename__ = 'adherence_data'
     id = db.Column(db.Integer, primary_key=True)
-    patient_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False)
+    patient_id = db.Column(
+        db.ForeignKey('users.id', ondelete='cascade'), index=True, nullable=False)
     rs_id_visit = db.Column(
         db.Text, index=True, nullable=False,
         doc="rs_id:visit_name")

portal/models/patient_list.py

@@ -0,0 +1,101 @@
+"""Module for PatientList, used specifically to populate and page patients"""
+from datetime import datetime, timedelta
+from ..database import db
+from .research_study import BASE_RS_ID, EMPRO_RS_ID
+
+
+class PatientList(db.Model):
+    """Maintain columns for all list fields, all indexed for quick sort
+
+    Table used to generate pages of results for patient lists.  Acts
+    as a cache, values should be updated on any change (questionnaire,
+    demographics, deletion, etc.)
+
+    All columns in both patients and sub-study lists are defined.
+    """
+    __tablename__ = 'patient_list'
+    userid = db.Column(
+        db.ForeignKey('users.id', ondelete='cascade'), primary_key=True, nullable=False)
+    study_id = db.Column(db.Text, index=True)
+    firstname = db.Column(db.String(64), index=True)
+    lastname = db.Column(db.String(64), index=True)
+    birthdate = db.Column(db.Date, index=True)
+    email = db.Column(db.String(120), index=True)
+    questionnaire_status = db.Column(db.Text, index=True)
+    empro_status = db.Column(db.Text, index=True)
+    clinician = db.Column(db.Text, index=True)
+    action_state = db.Column(db.Text, index=True)
+    visit = db.Column(db.Text, index=True)
+    empro_visit = db.Column(db.Text, index=True)
+    consentdate = db.Column(db.DateTime, index=True)
+    empro_consentdate = db.Column(db.DateTime, index=True)
+    org_name = db.Column(db.Text, index=True)
+    deleted = db.Column(db.Boolean, default=False)
+    test_role = db.Column(db.Boolean)
+    org_id = db.Column(db.ForeignKey('organizations.id'))  # used for access control
+    last_updated = db.Column(db.DateTime)
+
+
+def patient_list_update_patient(patient_id, research_study_id=None):
+    """Update given patient
+
+    :param research_study_id: define to optimize time for updating
+     only values from the given research_study_id.  by default, all columns
+     are (re)set to current info.
+    """
+    from .qb_timeline import qb_status_visit_name
+    from .role import ROLE
+    from .user import User
+    from .user_consent import consent_withdrawal_dates
+    from ..views.clinician import clinician_name_map
+
+    user = User.query.get(patient_id)
+    if not user.has_role(ROLE.PATIENT.value):
+        return
+
+    patient = PatientList.query.get(patient_id)
+    new_record = False
+    if not patient:
+        new_record = True
+        patient = PatientList(userid=patient_id)
+        db.session.add(patient)
+
+    if research_study_id is None or new_record:
+        patient.study_id = user.external_study_id
+        patient.firstname = user.first_name
+        patient.lastname = user.last_name
+        patient.email = user.email
+        patient.birthdate = user.birthdate
+        patient.deleted = user.deleted_id is not None
+        patient.test_role = True if user.has_role(ROLE.TEST.value) else False
+        patient.org_id = user.organizations[0].id if user.organizations else None
+        patient.org_name = user.organizations[0].name if user.organizations else None
+
+    # necessary to avoid recursive loop via some update paths
+    now = datetime.utcnow()
+    if patient.last_updated and patient.last_updated + timedelta(seconds=10) > now:
+        db.session.commit()
+        return
+
+    patient.last_updated = now
+    if research_study_id == BASE_RS_ID or research_study_id is None:
+        rs_id = BASE_RS_ID
+        qb_status = qb_status_visit_name(
+             patient.userid, research_study_id=rs_id, as_of_date=now)
+        patient.questionnaire_status = str(qb_status['status'])
+        patient.visit = qb_status['visit_name']
+        patient.consentdate, _ = consent_withdrawal_dates(user=user, research_study_id=rs_id)
+
+    if (research_study_id == EMPRO_RS_ID or research_study_id is None) and user.clinicians:
+        rs_id = EMPRO_RS_ID
+        patient.clinician = '; '.join(
+            (clinician_name_map().get(c.id, "not in map") for c in user.clinicians)) or ""
+        qb_status = qb_status_visit_name(
+            patient.userid, research_study_id=rs_id, as_of_date=now)
+        patient.empro_status = str(qb_status['status'])
+        patient.empro_visit = qb_status['visit_name']
+        patient.action_state = qb_status['action_state'].title() \
+            if qb_status['action_state'] else ""
+        patient.empro_consentdate, _ = consent_withdrawal_dates(
+            user=user, research_study_id=rs_id)
+    db.session.commit()

portal/models/reporting.py

@@ -6,6 +6,7 @@
 
 from flask import current_app
 from flask_babel import force_locale
+from flask_login import login_manager
 from werkzeug.exceptions import Unauthorized
 
 from ..audit import auditable_event
@@ -53,6 +54,10 @@ def single_patient_adherence_data(patient_id, research_study_id):
     if not patient.has_role(ROLE.PATIENT.value):
         return
 
+    # keep patient list data in sync
+    from .patient_list import patient_list_update_patient
+    patient_list_update_patient(patient_id=patient_id, research_study_id=research_study_id)
+
     as_of_date = datetime.utcnow()
     cache_moderation = CacheModeration(key=ADHERENCE_DATA_KEY.format(
         patient_id=patient_id,

portal/models/research_data.py

@@ -23,9 +23,11 @@ class ResearchData(db.Model):
     """
     __tablename__ = 'research_data'
     id = db.Column(db.Integer, primary_key=True)
-    subject_id = db.Column(db.ForeignKey('users.id'), index=True, nullable=False)
+    subject_id = db.Column(
+        db.ForeignKey('users.id', ondelete='cascade'), index=True, nullable=False)
     questionnaire_response_id = db.Column(
-        db.ForeignKey('questionnaire_responses.id'), index=True, unique=True, nullable=False,
+        db.ForeignKey('questionnaire_responses.id', ondelete='cascade'),
+        index=True, unique=True, nullable=False,
         doc="source questionnaire response")
     instrument = db.Column(db.Text, index=True, nullable=False)
     research_study_id = db.Column(db.Integer, index=True, nullable=False)