19875 - PAY Jobs - Disbursement Process handle Partial Refunds (#1428)

jobs/payment-jobs/pyproject.toml

@@ -8,7 +8,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.12"
 sbc-common-components = {git = "https://github.com/bcgov/sbc-common-components.git", subdirectory = "python"}
-pay-api = {git = "https://github.com/seeker25/sbc-pay.git", rev = "18263", subdirectory = "pay-api"}
+pay-api = {git = "https://github.com/Jxio/sbc-pay.git", rev = "19875", subdirectory = "pay-api"}
 flask-jwt-oidc = {git = "https://github.com/thorwolpert/flask-jwt-oidc.git"}
 simple-cloudevent = {git = "https://github.com/daxiom/simple-cloudevent.py.git"}
 gunicorn = "^21.2.0"

jobs/payment-jobs/tasks/ap_task.py

@@ -23,12 +23,12 @@
 from pay_api.models import DistributionCode as DistributionCodeModel
 from pay_api.models import EjvFile as EjvFileModel
 from pay_api.models import EjvHeader as EjvHeaderModel
-from pay_api.models import EjvInvoiceLink as EjvInvoiceLinkModel
+from pay_api.models import EjvLink as EjvLinkModel
 from pay_api.models import Invoice as InvoiceModel
 from pay_api.models import Refund as RefundModel
 from pay_api.models import RoutingSlip as RoutingSlipModel
 from pay_api.models import db
-from pay_api.utils.enums import DisbursementStatus, EjvFileType, RoutingSlipStatus
+from pay_api.utils.enums import DisbursementStatus, EjvFileType, EJVLinkType, RoutingSlipStatus
 from tasks.common.cgi_ap import CgiAP
 from tasks.common.dataclasses import APLine
 from tasks.ejv_partner_distribution_task import EjvPartnerDistributionTask
@@ -156,9 +156,10 @@ def _create_non_gov_disbursement_file(cls):  # pylint:disable=too-many-locals
             ap_content = f'{ap_content}{batch_trailer}'
 
             for inv in invoices:
-                db.session.add(EjvInvoiceLinkModel(invoice_id=inv.id,
-                                                   ejv_header_id=ejv_header_model.id,
-                                                   disbursement_status_code=DisbursementStatus.UPLOADED.value))
+                db.session.add(EjvLinkModel(link_id=inv.id,
+                                            link_type=EJVLinkType.INVOICE.value,
+                                            ejv_header_id=ejv_header_model.id,
+                                            disbursement_status_code=DisbursementStatus.UPLOADED.value))
                 inv.disbursement_status_code = DisbursementStatus.UPLOADED.value
             db.session.flush()
 

jobs/payment-jobs/tasks/eft_transfer_task.py

@@ -23,13 +23,14 @@
 from pay_api.models import EFTShortnames as EFTShortnameModel
 from pay_api.models import EjvFile as EjvFileModel
 from pay_api.models import EjvHeader as EjvHeaderModel
-from pay_api.models import EjvInvoiceLink as EjvInvoiceLinkModel
+from pay_api.models import EjvLink as EjvLinkModel
 from pay_api.models import Invoice as InvoiceModel
 from pay_api.models import PaymentAccount as PaymentAccountModel
 from pay_api.models import PaymentLineItem as PaymentLineItemModel
 from pay_api.models import db
 from pay_api.services.flags import flags
-from pay_api.utils.enums import DisbursementStatus, EFTGlTransferType, EjvFileType, InvoiceStatus, PaymentMethod
+from pay_api.utils.enums import (
+    DisbursementStatus, EFTGlTransferType, EjvFileType, EJVLinkType, InvoiceStatus, PaymentMethod)
 from sqlalchemy import exists, func
 
 from tasks.common.cgi_ejv import CgiEjv
@@ -173,9 +174,10 @@ def process_invoice_ejv_links(invoices: List[InvoiceModel], ejv_header_model_id:
         for inv in invoices:
             current_app.logger.debug(f'Creating EJV Invoice Link for invoice id: {inv.id}')
             # Create Ejv file link and flush
-            ejv_invoice_link = EjvInvoiceLinkModel(invoice_id=inv.id, ejv_header_id=ejv_header_model_id,
-                                                   disbursement_status_code=DisbursementStatus.UPLOADED.value,
-                                                   sequence=sequence)
+            ejv_invoice_link = EjvLinkModel(link_id=inv.id, link_type=EJVLinkType.INVOICE.value,
+                                            ejv_header_id=ejv_header_model_id,
+                                            disbursement_status_code=DisbursementStatus.UPLOADED.value,
+                                            sequence=sequence)
             db.session.add(ejv_invoice_link)
             sequence += 1
 

jobs/payment-jobs/tasks/ejv_partner_distribution_task.py

@@ -23,13 +23,14 @@
 from pay_api.models import DistributionCodeLink as DistributionCodeLinkModel
 from pay_api.models import EjvFile as EjvFileModel
 from pay_api.models import EjvHeader as EjvHeaderModel
-from pay_api.models import EjvInvoiceLink as EjvInvoiceLinkModel
+from pay_api.models import EjvLink as EjvLinkModel
 from pay_api.models import FeeSchedule as FeeScheduleModel
 from pay_api.models import Invoice as InvoiceModel
 from pay_api.models import PaymentLineItem as PaymentLineItemModel
+from pay_api.models import RefundsPartial as RefundsPartialModel
 from pay_api.models import Receipt as ReceiptModel
 from pay_api.models import db
-from pay_api.utils.enums import DisbursementStatus, EjvFileType, InvoiceStatus, PaymentMethod
+from pay_api.utils.enums import DisbursementStatus, EjvFileType, EJVLinkType, InvoiceStatus, PaymentMethod
 from sqlalchemy import Date, cast
 
 from tasks.common.cgi_ejv import CgiEjv
@@ -68,6 +69,21 @@ def get_invoices_for_disbursement(partner):
         current_app.logger.info(invoices)
         return invoices
 
+    @staticmethod
+    def get_refund_partial_payment_line_items_for_disbursement(partner) -> List[PaymentLineItemModel]:
+        """Return payment line items with partial refunds for disbursement."""
+        payment_line_items: List[PaymentLineItemModel] = db.session.query(PaymentLineItemModel) \
+            .join(InvoiceModel, PaymentLineItemModel.invoice_id == InvoiceModel.id) \
+            .join(RefundsPartialModel, PaymentLineItemModel.id == RefundsPartialModel.payment_line_item_id) \
+            .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \
+            .filter(InvoiceModel.payment_method_code.in_([PaymentMethod.DIRECT_PAY.value])) \
+            .filter((RefundsPartialModel.disbursement_status_code.is_(None)) |
+                    (RefundsPartialModel.disbursement_status_code == DisbursementStatus.ERRORED.value)) \
+            .filter(InvoiceModel.corp_type_code == partner.code) \
+            .all()
+        current_app.logger.info(payment_line_items)
+        return payment_line_items
+
     @classmethod
     def get_invoices_for_refund_reversal(cls, partner):
         """Return invoices for refund reversal."""
@@ -113,12 +129,16 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
 
         for partner in partners:
             # Find all invoices for the partner to disburse.
-            # This includes invoices which are not PAID and invoices which are refunded.
+            # This includes invoices which are not PAID and invoices which are refunded and partial refunded.
             payment_invoices = cls.get_invoices_for_disbursement(partner)
             refund_reversals = cls.get_invoices_for_refund_reversal(partner)
             invoices = payment_invoices + refund_reversals
+
+            # Process partial refunds for each partner
+            refund_partial_items = cls.get_refund_partial_payment_line_items_for_disbursement(partner)
+
             # If no invoices continue.
-            if not invoices:
+            if not invoices and not refund_partial_items:
                 continue
 
             effective_date: str = cls.get_effective_date()
@@ -134,11 +154,16 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
             # and create one JV Header and detail for each.
             distribution_code_set = set()
             invoice_id_list = []
+            partial_line_item_id_list = []
             for inv in invoices:
                 invoice_id_list.append(inv.id)
                 for line_item in inv.payment_line_items:
                     distribution_code_set.add(line_item.fee_distribution_id)
 
+            for line_item in refund_partial_items:
+                partial_line_item_id_list.append(line_item.id)
+                distribution_code_set.add(line_item.fee_distribution_id)
+
             for distribution_code_id in list(distribution_code_set):
                 distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id(distribution_code_id)
                 credit_distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id(
@@ -147,13 +172,22 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
                 if credit_distribution_code.stop_ejv:
                     continue
 
-                line_items = cls._find_line_items_by_invoice_and_distribution(distribution_code_id, invoice_id_list)
+                line_items = cls._find_line_items_by_invoice_and_distribution(
+                        distribution_code_id, invoice_id_list)
+
+                refund_partial_items = cls._find_refund_partial_items_by_distribution(
+                        distribution_code_id, partial_line_item_id_list)
 
                 total: float = 0
                 for line in line_items:
                     total += line.total
 
+                partial_refund_total: float = 0
+                for refund_partial in refund_partial_items:
+                    partial_refund_total += refund_partial.refund_amount
+
                 batch_total += total
+                batch_total += partial_refund_total
 
                 debit_distribution = cls.get_distribution_string(distribution_code)  # Debit from BCREG GL
                 credit_distribution = cls.get_distribution_string(credit_distribution_code)  # Credit to partner GL
@@ -193,20 +227,39 @@ def _create_ejv_file_for_partner(cls, batch_type: str):  # pylint:disable=too-ma
 
                     control_total += 1
 
+                partial_refund_number: int = 0
+                for refund_partial in refund_partial_items:
+                    # JV Details for partial refunds
+                    partial_refund_number += 1
+                    # Flow Through add it as the refunds_partial id.
+                    flow_through = f'{refund_partial.id:<110}'
+                    refund_partial_number = f'#{refund_partial.id}'
+                    description = disbursement_desc[:-len(refund_partial_number)] + refund_partial_number
+                    description = f'{description[:100]:<100}'
+
+                    ejv_content = '{}{}'.format(ejv_content,  # pylint:disable=consider-using-f-string
+                                                cls.get_jv_line(batch_type, credit_distribution, description,
+                                                                effective_date, flow_through, journal_name,
+                                                                refund_partial.refund_amount,
+                                                                partial_refund_number, 'D'))
+                    partial_refund_number += 1
+                    control_total += 1
+
+                    # Add a line here for debit too
+                    ejv_content = '{}{}'.format(ejv_content,  # pylint:disable=consider-using-f-string
+                                                cls.get_jv_line(batch_type, debit_distribution, description,
+                                                                effective_date, flow_through, journal_name,
+                                                                refund_partial.refund_amount,
+                                                                partial_refund_number, 'C'))
+                    control_total += 1
+
+                    # Update partial refund status
+                    refund_partial.disbursement_status_code = DisbursementStatus.UPLOADED.value
+
+            # Create ejv invoice/partial_refund link records and set invoice status
             sequence = 1
-            # Create ejv invoice link records and set invoice status
-            for inv in invoices:
-                # Create Ejv file link and flush
-                link_model = EjvInvoiceLinkModel(invoice_id=inv.id,
-                                                 ejv_header_id=ejv_header_model.id,
-                                                 disbursement_status_code=DisbursementStatus.UPLOADED.value,
-                                                 sequence=sequence)
-                # Set distribution status to invoice
-                db.session.add(link_model)
-                sequence += 1
-                inv.disbursement_status_code = DisbursementStatus.UPLOADED.value
-
-            db.session.flush()
+            sequence = cls._create_ejv_link(invoices, ejv_header_model, sequence, EJVLinkType.INVOICE.value)
+            cls._create_ejv_link(refund_partial_items, ejv_header_model, sequence, EJVLinkType.REFUND.value)
 
         if not ejv_content:
             db.session.rollback()
@@ -238,6 +291,18 @@ def _find_line_items_by_invoice_and_distribution(cls, distribution_code_id, invo
             .filter(PaymentLineItemModel.fee_distribution_id == distribution_code_id)
         return line_items
 
+    @classmethod
+    def _find_refund_partial_items_by_distribution(cls, distribution_code_id, partial_line_item_id_list) \
+            -> List[RefundsPartialModel]:
+        """Find and return all payment line items for this distribution."""
+        line_items: List[RefundsPartialModel] = db.session.query(RefundsPartialModel) \
+            .join(PaymentLineItemModel, PaymentLineItemModel.id == RefundsPartialModel.payment_line_item_id) \
+            .filter(RefundsPartialModel.payment_line_item_id.in_(partial_line_item_id_list)) \
+            .filter(RefundsPartialModel.refund_amount > 0) \
+            .filter(PaymentLineItemModel.fee_distribution_id == distribution_code_id) \
+            .all()
+        return line_items
+
     @classmethod
     def _get_partners_by_batch_type(cls, batch_type) -> List[CorpTypeModel]:
         """Return partners by batch type."""
@@ -272,3 +337,17 @@ def _get_partners_by_batch_type(cls, batch_type) -> List[CorpTypeModel]:
         corp_type_codes: List[str] = db.session.scalars(corp_type_query).all()
 
         return db.session.query(CorpTypeModel).filter(CorpTypeModel.code.in_(corp_type_codes)).all()
+
+    @classmethod
+    def _create_ejv_link(cls, items, ejv_header_model, sequence, link_type):
+        for item in items:
+            link_model = EjvLinkModel(link_id=item.id,
+                                      link_type=link_type,
+                                      ejv_header_id=ejv_header_model.id,
+                                      disbursement_status_code=DisbursementStatus.UPLOADED.value,
+                                      sequence=sequence)
+            db.session.add(link_model)
+            sequence += 1
+            item.disbursement_status_code = DisbursementStatus.UPLOADED.value
+        db.session.flush()
+        return sequence

jobs/payment-jobs/tasks/ejv_payment_task.py

@@ -20,12 +20,13 @@
 from pay_api.models import DistributionCode as DistributionCodeModel
 from pay_api.models import EjvFile as EjvFileModel
 from pay_api.models import EjvHeader as EjvHeaderModel
-from pay_api.models import EjvInvoiceLink as EjvInvoiceLinkModel
+from pay_api.models import EjvLink as EjvLinkModel
 from pay_api.models import Invoice as InvoiceModel
 from pay_api.models import InvoiceReference as InvoiceReferenceModel
 from pay_api.models import PaymentAccount as PaymentAccountModel
 from pay_api.models import db
-from pay_api.utils.enums import DisbursementStatus, EjvFileType, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod
+from pay_api.utils.enums import (
+    DisbursementStatus, EjvFileType, EJVLinkType, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod)
 from pay_api.utils.util import generate_transaction_number
 
 from tasks.common.cgi_ejv import CgiEjv
@@ -180,10 +181,10 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str):  # pylint:disable=to
             sequence = 1
             for inv in invoices:
                 current_app.logger.debug(f'Creating EJV Invoice Link for invoice id: {inv.id}')
-                # Create Ejv file link and flush
-                ejv_invoice_link = EjvInvoiceLinkModel(invoice_id=inv.id, ejv_header_id=ejv_header_model.id,
-                                                       disbursement_status_code=DisbursementStatus.UPLOADED.value,
-                                                       sequence=sequence)
+                ejv_invoice_link = EjvLinkModel(link_id=inv.id, link_type=EJVLinkType.INVOICE.value,
+                                                ejv_header_id=ejv_header_model.id,
+                                                disbursement_status_code=DisbursementStatus.UPLOADED.value,
+                                                sequence=sequence)
                 db.session.add(ejv_invoice_link)
                 sequence += 1
                 # Set distribution status to invoice

jobs/payment-jobs/tests/jobs/factory.py

@@ -21,7 +21,8 @@
 
 from pay_api.models import (
     CfsAccount, DistributionCode, DistributionCodeLink, EFTShortnames, Invoice, InvoiceReference, Payment,
-    PaymentAccount, PaymentLineItem, Receipt, Refund, RoutingSlip, StatementRecipients, StatementSettings)
+    PaymentAccount, PaymentLineItem, Receipt, Refund, RefundsPartial, RoutingSlip, StatementRecipients,
+    StatementSettings)
 from pay_api.utils.enums import (
     CfsAccountStatus, InvoiceReferenceStatus, InvoiceStatus, LineItemStatus, PaymentMethod, PaymentStatus,
     PaymentSystem, RoutingSlipStatus)
@@ -334,3 +335,20 @@ def factory_refund_invoice(
         requested_by='TEST',
         details=details
     ).save()
+
+
+def factory_refund_partial(
+        payment_line_item_id: int,
+        refund_amount: float,
+        refund_type: str,
+        created_by='test',
+        created_on: datetime = datetime.now()
+):
+    """Return Factory."""
+    return RefundsPartial(
+        payment_line_item_id=payment_line_item_id,
+        refund_amount=refund_amount,
+        refund_type=refund_type,
+        created_by=created_by,
+        created_on=created_on
+    ).save()

jobs/payment-jobs/tests/jobs/test_eft_transfer_task.py

@@ -19,7 +19,7 @@
 from datetime import datetime
 from typing import List
 
-from pay_api.models import DistributionCode, EFTGLTransfer, EjvFile, EjvHeader, EjvInvoiceLink, FeeSchedule, Invoice, db
+from pay_api.models import DistributionCode, EFTGLTransfer, EjvFile, EjvHeader, EjvLink, FeeSchedule, Invoice, db
 from pay_api.utils.enums import DisbursementStatus, EFTGlTransferType, EjvFileType, InvoiceStatus, PaymentMethod
 
 from tasks.eft_transfer_task import EftTransferTask
@@ -89,8 +89,8 @@ def test_eft_transfer(app, session, monkeypatch):
 
     # Lookup invoice and assert disbursement status
     for invoice in invoices:
-        ejv_inv_link: EjvInvoiceLink = db.session.query(EjvInvoiceLink) \
-            .filter(EjvInvoiceLink.invoice_id == invoice.id).first()
+        ejv_inv_link: EjvLink = db.session.query(EjvLink) \
+            .filter(EjvLink.link_id == invoice.id).first()
         assert ejv_inv_link
 
         ejv_header = db.session.query(EjvHeader).filter(EjvHeader.id == ejv_inv_link.ejv_header_id).first()

jobs/payment-jobs/tests/jobs/test_ejv_partner_distribution_task.py

@@ -22,7 +22,7 @@
 from flask import current_app
 from freezegun import freeze_time
 from pay_api.models import CorpType as CorpTypeModel
-from pay_api.models import DistributionCode, EjvFile, EjvHeader, EjvInvoiceLink, FeeSchedule, Invoice, db
+from pay_api.models import DistributionCode, EjvFile, EjvHeader, EjvLink, FeeSchedule, Invoice, db
 from pay_api.utils.enums import CfsAccountStatus, DisbursementStatus, InvoiceStatus, PaymentMethod
 
 from tasks.ejv_partner_distribution_task import EjvPartnerDistributionTask
@@ -89,7 +89,7 @@ def test_disbursement_for_partners(session, monkeypatch, client_code, batch_type
         invoice = Invoice.find_by_id(invoice.id)
         assert invoice.disbursement_status_code == DisbursementStatus.UPLOADED.value
 
-        ejv_inv_link = db.session.query(EjvInvoiceLink).filter(EjvInvoiceLink.invoice_id == invoice.id).first()
+        ejv_inv_link = db.session.query(EjvLink).filter(EjvLink.link_id == invoice.id).first()
         assert ejv_inv_link
 
         ejv_header = db.session.query(EjvHeader).filter(EjvHeader.id == ejv_inv_link.ejv_header_id).first()