Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance tweak statements #1145

Merged
merged 10 commits into from
Mar 17, 2023
Merged

Performance tweak statements #1145

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1a0043c
Bulk save statement_invoices.
seeker25 Mar 17, 2023
a6ad9e2
Add in additional logging.
seeker25 Mar 17, 2023
cf4324c
remove f strings
seeker25 Mar 17, 2023
2009111
Change delete to use SQL instead of ORM. Weekly deletes with 25000+ r…
seeker25 Mar 17, 2023
e59f8d7
Fix array.
seeker25 Mar 17, 2023
e81d424
unit test fix
seeker25 Mar 17, 2023
b89c081
Move to more code based approach.
seeker25 Mar 17, 2023
94e04e1
Modify delete some more.
seeker25 Mar 17, 2023
648b56f
Remove execution options, it's a sqlalchemy 2.0 feature.
seeker25 Mar 17, 2023
3da796a
Redundant line.
seeker25 Mar 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions jobs/payment-jobs/tasks/statement_task.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from pay_api.utils.util import (
get_first_and_last_dates_of_month, get_local_time, get_previous_day, get_previous_month_and_year,
get_week_start_and_end_date)
from sqlalchemy import delete


class StatementTask: # pylint:disable=too-few-public-methods
Expand Down Expand Up @@ -129,6 +130,7 @@ def _create_statement_records(cls, search_filter, statement_settings):
invoices_and_auth_ids = PaymentModel.get_invoices_for_statements(search_filter)
if cls.has_date_override:
cls._clean_up_old_statements(statement_settings, statement_from, statement_to)
current_app.logger.debug('Inserting statements.')
statements = [StatementModel(
frequency=setting.frequency,
statement_settings_id=setting.id,
Expand All @@ -144,14 +146,15 @@ def _create_statement_records(cls, search_filter, statement_settings):
db.session.bulk_save_objects(statements, return_defaults=True)
db.session.flush()

current_app.logger.debug('Inserting statement invoices.')
statement_invoices = []
for statement, auth_account_id in zip(statements, auth_account_ids):
invoices = [i for i in invoices_and_auth_ids if i.auth_account_id == auth_account_id]
for invoice in invoices:
statement_invoice = StatementInvoicesModel(
statement_id=statement.id,
invoice_id=invoice.id
)
db.session.add(statement_invoice)
statement_invoices = statement_invoices + [StatementInvoicesModel(
statement_id=statement.id,
invoice_id=invoice.id
) for invoice in invoices]
db.session.bulk_save_objects(statement_invoices)

@classmethod
def _clean_up_old_statements(cls, statement_settings, statement_from, statement_to):
Expand All @@ -168,8 +171,10 @@ def _clean_up_old_statements(cls, statement_settings, statement_from, statement_
remove_statement_invoices = db.session.query(StatementInvoicesModel)\
.filter(StatementInvoicesModel.statement_id.in_(remove_statements_ids))\
.all()
for statement_invoices in remove_statement_invoices:
db.session.delete(statement_invoices)
statement_invoice_ids = [statement_invoice.id for statement_invoice in remove_statement_invoices]
delete_statement_invoice = delete(StatementInvoicesModel)\
.where(StatementInvoicesModel.id.in_(statement_invoice_ids))
db.session.execute(delete_statement_invoice)
db.session.flush()
for statement in remove_statements:
db.session.delete(statement)
delete_statement = delete(StatementModel).where(StatementModel.id.in_(remove_statements_ids))
db.session.execute(delete_statement)