diff --git a/jobs/payment-jobs/logging.conf b/jobs/payment-jobs/logging.conf index 0806a8a2c..00ca98c72 100644 --- a/jobs/payment-jobs/logging.conf +++ b/jobs/payment-jobs/logging.conf @@ -1,5 +1,5 @@ [loggers] -keys=root,api +keys=root,api,invoke_jobs [handlers] keys=console @@ -17,6 +17,12 @@ handlers=console qualname=api propagate=0 +[logger_invoke_jobs] +level=DEBUG +handlers=console +qualname=invoke_jobs +propagate=0 + [handler_console] class=StreamHandler level=DEBUG diff --git a/jobs/payment-jobs/requirements.txt b/jobs/payment-jobs/requirements.txt index 34cd66f39..9c32e2ce2 100644 --- a/jobs/payment-jobs/requirements.txt +++ b/jobs/payment-jobs/requirements.txt @@ -1,5 +1,5 @@ -e git+https://github.com/bcgov/sbc-common-components.git@a8a2074b12dbf36cc23634206e37d57ccfa5a33d#egg=sbc_common_components&subdirectory=python --e git+https://github.com/bcgov/sbc-pay.git@26bd80b7a07c1d5c48433a1be58f18152277af56#egg=pay_api&subdirectory=pay-api +-e git+https://github.com/bcgov/sbc-pay.git@858402b14285103f9c2f168d9541531893184b04#egg=pay_api&subdirectory=pay-api Flask-Caching==2.0.2 Flask-Migrate==2.7.0 Flask-Moment==1.0.5 @@ -10,6 +10,7 @@ Flask==1.1.2 Jinja2==3.0.3 Mako==1.2.4 MarkupSafe==2.1.2 +PyMeeus==0.5.12 PyNaCl==1.5.0 SQLAlchemy-Continuum==1.3.14 SQLAlchemy-Utils==0.40.0 @@ -20,6 +21,7 @@ aniso8601==9.0.1 asyncio-nats-client==0.11.5 asyncio-nats-streaming==0.4.0 attrs==22.2.0 +backports.zoneinfo==0.2.1 bcrypt==4.0.1 blinker==1.5 cachelib==0.9.0 @@ -28,6 +30,7 @@ certifi==2022.12.7 cffi==1.15.1 charset-normalizer==3.1.0 click==8.1.3 +convertdate==2.4.0 croniter==1.3.8 cryptography==40.0.1 cx-Oracle==8.3.0 @@ -40,12 +43,15 @@ flask-jwt-oidc==0.3.0 flask-marshmallow==0.11.0 flask-restx==1.1.0 gunicorn==20.1.0 +hijri-converter==2.2.4 +holidays==0.21.13 idna==3.4 importlib-metadata==6.1.0 importlib-resources==5.12.0 itsdangerous==2.0.1 jaeger-client==4.8.0 jsonschema==4.17.3 +korean-lunar-calendar==0.3.1 launchdarkly-server-sdk==8.1.1 marshmallow-sqlalchemy==0.25.0 marshmallow==3.19.0 @@ -69,7 +75,7 @@ pytz==2023.2 requests==2.28.2 rsa==4.9 semver==2.13.0 -sentry-sdk==1.17.0 +sentry-sdk==1.18.0 six==1.16.0 threadloop==1.0.2 thrift==0.16.0 diff --git a/jobs/payment-jobs/requirements/prod.txt b/jobs/payment-jobs/requirements/prod.txt index f581cf5ad..d544fd870 100644 --- a/jobs/payment-jobs/requirements/prod.txt +++ b/jobs/payment-jobs/requirements/prod.txt @@ -18,3 +18,5 @@ dataclass_wizard launchdarkly-server-sdk cx_Oracle more_itertools +holidays +pytz diff --git a/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py b/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py index 2e585a4eb..d3f10a929 100644 --- a/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py +++ b/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py @@ -33,6 +33,7 @@ from sqlalchemy import Date, cast from tasks.common.cgi_ejv import CgiEjv +from utils.date import is_holiday class EjvPartnerDistributionTask(CgiEjv): @@ -49,6 +50,10 @@ def create_ejv_file(cls): 4. Upload to sftp for processing. First upload JV file and then a TRG file. 5. Update the statuses and create records to for the batch. """ + if is_holiday(): + current_app.logger.info('Deferring ejv disbursement task for another day.') + return + cls._create_ejv_file_for_partner(batch_type='GI') # Internal ministry cls._create_ejv_file_for_partner(batch_type='GA') # External ministry diff --git a/jobs/payment-jobs/tasks/ejv_payment_task.py b/jobs/payment-jobs/tasks/ejv_payment_task.py index 900555fa8..292628eec 100644 --- a/jobs/payment-jobs/tasks/ejv_payment_task.py +++ b/jobs/payment-jobs/tasks/ejv_payment_task.py @@ -29,6 +29,7 @@ from pay_api.utils.util import generate_transaction_number from tasks.common.cgi_ejv import CgiEjv +from utils.date import is_holiday class EjvPaymentTask(CgiEjv): @@ -46,6 +47,10 @@ def create_ejv_file(cls): 5. Upload to sftp for processing. First upload JV file and then a TRG file. 6. Update the statuses and create records to for the batch. """ + if is_holiday(): + current_app.logger.info('Deferring ejv payment task for another day.') + return + cls._create_ejv_file_for_gov_account(batch_type='GI') cls._create_ejv_file_for_gov_account(batch_type='GA') diff --git a/jobs/payment-jobs/tests/jobs/conftest.py b/jobs/payment-jobs/tests/jobs/conftest.py index 68825d8cc..c496ae842 100644 --- a/jobs/payment-jobs/tests/jobs/conftest.py +++ b/jobs/payment-jobs/tests/jobs/conftest.py @@ -25,6 +25,7 @@ from sqlalchemy.schema import DropConstraint, MetaData from invoke_jobs import create_app +from utils.logger import setup_logging @pytest.fixture(scope='session') @@ -118,6 +119,8 @@ def db(app): # pylint: disable=redefined-outer-name, invalid-name Migrate(app, _db, directory=migrations_path) upgrade() + # Restore the logging, alembic and sqlalchemy have their own logging from alembic.ini. + setup_logging(os.path.abspath('logging.conf')) return _db diff --git a/jobs/payment-jobs/tests/jobs/test_ejv_payment_task.py b/jobs/payment-jobs/tests/jobs/test_ejv_payment_task.py index 5eae96032..26b4c875c 100644 --- a/jobs/payment-jobs/tests/jobs/test_ejv_payment_task.py +++ b/jobs/payment-jobs/tests/jobs/test_ejv_payment_task.py @@ -16,6 +16,8 @@ Test-Suite to ensure that the CgiEjvJob is working as expected. """ +import unittest +from freezegun import freeze_time from pay_api.models import ( DistributionCode, EjvFile, EjvHeader, EjvInvoiceLink, FeeSchedule, Invoice, InvoiceReference, db) from pay_api.utils.enums import DisbursementStatus, EjvFileType, InvoiceReferenceStatus, InvoiceStatus @@ -137,3 +139,25 @@ def test_payments_for_gov_accounts(session, monkeypatch): ejv_file: EjvFile = EjvFile.find_by_id(ejv_header.ejv_file_id) assert ejv_file assert ejv_file.file_type == EjvFileType.PAYMENT.value + + +def test_ejv_skip_holidays(session): + """Assert that the EJV job skips holidays.""" + test_case = unittest.TestCase() + # Christmas + with test_case.assertLogs('invoke_jobs') as log: + with freeze_time('2023-12-25 00:00:00T08:00:00'): + EjvPaymentTask.create_ejv_file() + assert [message for message in log.output if 'stat holiday' in message] + + # New Years + with test_case.assertLogs('invoke_jobs') as log: + with freeze_time('2024-01-01 00:00:00T08:00:00'): + EjvPaymentTask.create_ejv_file() + assert [message for message in log.output if 'stat holiday' in message] + + # Labour Day + with test_case.assertLogs('invoke_jobs') as log: + with freeze_time('2023-09-04 00:00:00T08:00:00'): + EjvPaymentTask.create_ejv_file() + assert [message for message in log.output if 'stat holiday' in message] diff --git a/jobs/payment-jobs/utils/date.py b/jobs/payment-jobs/utils/date.py new file mode 100644 index 000000000..fee91613b --- /dev/null +++ b/jobs/payment-jobs/utils/date.py @@ -0,0 +1,13 @@ + +from datetime import datetime +import holidays +import pytz + +from flask import current_app + +def is_holiday(): + current_date_pacific = datetime.now(tz=pytz.utc).astimezone(pytz.timezone('US/Pacific')).strftime('%Y-%m-%d') + if holiday := holidays.CA(state='BC', observed=False).get(current_date_pacific): + current_app.logger.info(f'Today is a stat holiday {holiday} on {current_date_pacific}') + return True + return False diff --git a/jobs/payment-jobs/utils/logger.py b/jobs/payment-jobs/utils/logger.py index 9c737649a..8b88ddcf2 100755 --- a/jobs/payment-jobs/utils/logger.py +++ b/jobs/payment-jobs/utils/logger.py @@ -26,6 +26,6 @@ def setup_logging(conf): if conf and path.isfile(conf): logging.config.fileConfig(conf) - print('Configure logging, from conf:{}'.format(conf), file=sys.stdout) + print(f'Configure logging, from conf:{conf}', file=sys.stdout) else: - print('Unable to configure logging, attempted conf:{}'.format(conf), file=sys.stderr) + print(f'Unable to configure logging, attempted conf:{conf}', file=sys.stderr) diff --git a/pay-api/src/pay_api/services/fee_schedule.py b/pay-api/src/pay_api/services/fee_schedule.py index e29e8f1e4..f3bd6cb71 100644 --- a/pay-api/src/pay_api/services/fee_schedule.py +++ b/pay-api/src/pay_api/services/fee_schedule.py @@ -24,7 +24,6 @@ from pay_api.models import FeeCode as FeeCodeModel from pay_api.models import FeeSchedule as FeeScheduleModel from pay_api.models import FeeScheduleSchema -from pay_api.services.flags import flags from pay_api.utils.enums import Role from pay_api.utils.errors import Error from pay_api.utils.user_context import UserContext, user_context @@ -312,11 +311,6 @@ def find_by_corp_type_and_filing_type( # pylint: disable=too-many-arguments # Set transaction fees fee_schedule.service_fees = FeeSchedule.calculate_service_fees(fee_schedule_dao, account_fee) - # Special case for CSO partner type which is different from normal flow - if flags.is_on('BAD_CSO_SERVICE_FEE', default=True) and fee_schedule.corp_type_code == 'CSO' \ - and fee_schedule.quantity: - fee_schedule.service_fees = fee_schedule.service_fees * fee_schedule.quantity - if kwargs.get('is_priority') and fee_schedule_dao.priority_fee and apply_filing_fees: fee_schedule.priority_fee = fee_schedule_dao.priority_fee.amount if kwargs.get('is_future_effective') and fee_schedule_dao.future_effective_fee and apply_filing_fees: diff --git a/queue_services/payment-reconciliations/requirements.txt b/queue_services/payment-reconciliations/requirements.txt index 4c83bd8c5..cfb4108c2 100644 --- a/queue_services/payment-reconciliations/requirements.txt +++ b/queue_services/payment-reconciliations/requirements.txt @@ -1,6 +1,6 @@ --e git+https://github.com/bcgov/lear.git@1817e8ea1d43054845384125cdcb92273f1b53c9#egg=entity_queue_common&subdirectory=queue_services/common --e git+https://github.com/bcgov/sbc-common-components.git@a643801c373063dfa44a2f7213d9122d3ad03851#egg=sbc_common_components&subdirectory=python --e git+https://github.com/bcgov/sbc-pay.git@e722e68450ac28e9c5f3352b10edce2d0388c327#egg=pay_api&subdirectory=pay-api +-e git+https://github.com/bcgov/lear.git@757ebe62916fe4e9467cf8505ddd41e15ae3282e#egg=entity_queue_common&subdirectory=queue_services/common +-e git+https://github.com/bcgov/sbc-common-components.git@a8a2074b12dbf36cc23634206e37d57ccfa5a33d#egg=sbc_common_components&subdirectory=python +-e git+https://github.com/bcgov/sbc-pay.git@858402b14285103f9c2f168d9541531893184b04#egg=pay_api&subdirectory=pay-api Flask-Caching==2.0.2 Flask-Migrate==2.7.0 Flask-Moment==1.0.5 @@ -17,7 +17,7 @@ SQLAlchemy==1.3.24 Werkzeug==1.0.1 aiohttp==3.8.4 aiosignal==1.3.1 -alembic==1.9.4 +alembic==1.10.2 aniso8601==9.0.1 async-timeout==4.0.2 asyncio-nats-client==0.11.5 @@ -25,22 +25,24 @@ asyncio-nats-streaming==0.4.0 attrs==22.2.0 blinker==1.5 cachelib==0.9.0 +cattrs==22.2.0 certifi==2022.12.7 cffi==1.15.1 -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 click==8.1.3 croniter==1.3.8 -cryptography==39.0.1 -dpath==2.1.4 +cryptography==40.0.1 +dpath==2.1.5 ecdsa==0.18.0 +exceptiongroup==1.1.1 expiringdict==1.2.2 flask-jwt-oidc==0.3.0 flask-marshmallow==0.11.0 -flask-restx==1.0.6 +flask-restx==1.1.0 frozenlist==1.3.3 gunicorn==20.1.0 idna==3.4 -importlib-metadata==6.0.0 +importlib-metadata==6.1.0 importlib-resources==5.12.0 itsdangerous==2.0.1 jaeger-client==4.8.0 @@ -48,7 +50,7 @@ jsonschema==4.17.3 launchdarkly-server-sdk==8.1.1 marshmallow-sqlalchemy==0.25.0 marshmallow==3.19.0 -minio==7.1.13 +minio==7.1.14 multidict==6.0.4 opentracing==2.4.0 packaging==23.0 @@ -61,17 +63,18 @@ pycountry==22.3.5 pycparser==2.21 pyrsistent==0.19.3 python-dateutil==2.8.2 -python-dotenv==0.21.1 +python-dotenv==1.0.0 python-jose==3.3.0 -pytz==2022.7.1 +pytz==2023.2 requests==2.28.2 rsa==4.9 semver==2.13.0 -sentry-sdk==1.15.0 +sentry-sdk==1.18.0 six==1.16.0 threadloop==1.0.2 thrift==0.16.0 tornado==6.2 -urllib3==1.26.14 +typing_extensions==4.5.0 +urllib3==1.26.15 yarl==1.8.2 -zipp==3.14.0 +zipp==3.15.0