Skip to content

Commit

Permalink
Max retry for exports (#662)
Browse files Browse the repository at this point in the history 
* Max retry for exports

* proper tests

* remove loggers

* remove loggers

* flake resolved
  • Loading branch information
@Ashutosh619-sudo
Ashutosh619-sudo authored Aug 29, 2024
1 parent 349e2f7 commit a83a2c1
Show file tree
Hide file tree
Showing 3 changed files with 266 additions and 20 deletions.
66 changes: 59 additions & 7 deletions apps/quickbooks_online/queue.py
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,36 @@
from datetime import datetime, timedelta
import logging
from datetime import datetime, timedelta, timezone
from typing import List

from django.db.models import Q
from django_q.models import Schedule
from django_q.tasks import Chain, async_task

from apps.fyle.models import Expense, ExpenseGroup
from apps.tasks.models import TaskLog
from apps.tasks.models import TaskLog, Error
from apps.workspaces.models import FyleCredential, WorkspaceGeneralSettings


logger = logging.getLogger(__name__)
logger.level = logging.INFO


def async_run_post_configration_triggers(workspace_general_settings: WorkspaceGeneralSettings):
async_task('apps.quickbooks_online.tasks.async_sync_accounts', int(workspace_general_settings.workspace_id), q_options={'cluster': 'import'})


def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error):
"""
Validate failing export
:param is_auto_export: Is auto export
:param interval_hours: Interval hours
:param error: Error
"""
# If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day
return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24)


def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule bills creation
:param expense_group_ids: List of expense group ids
Expand All @@ -26,10 +42,17 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_
if expense_group_ids:
expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, bill__id__isnull=True, exported_at__isnull=True).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue
task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_BILL'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_BILL'
Expand Down Expand Up @@ -79,7 +102,7 @@ def __create_chain_and_run(fyle_credentials: FyleCredential, in_progress_expense
chain.run()


def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule cheque creation
:param expense_group_ids: List of expense group ids
Expand All @@ -91,10 +114,17 @@ def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], i
if expense_group_ids:
expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, cheque__id__isnull=True, exported_at__isnull=True).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue
task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_CHECK'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_CHECK'
Expand All @@ -121,7 +151,7 @@ def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], i
__create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source)


def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule journal_entry creation
:param expense_group_ids: List of expense group ids
Expand All @@ -133,10 +163,17 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s
Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, journalentry__id__isnull=True, exported_at__isnull=True
).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue
task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_JOURNAL_ENTRY'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_JOURNAL_ENTRY'
Expand All @@ -162,7 +199,7 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s
__create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source)


def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule credit card purchase creation
:param expense_group_ids: List of expense group ids
Expand All @@ -176,10 +213,18 @@ def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids:
Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, creditcardpurchase__id__isnull=True, exported_at__isnull=True
).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue

task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_CREDIT_CARD_PURCHASE'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_CREDIT_CARD_PURCHASE'
Expand All @@ -206,7 +251,7 @@ def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids:
__create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source)


def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule QBO expense creation
:param expense_group_ids: List of expense group ids
Expand All @@ -217,11 +262,18 @@ def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str
"""
if expense_group_ids:
expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, qboexpense__id__isnull=True, exported_at__isnull=True).all()
errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue

task_log, _ = TaskLog.objects.get_or_create(
workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_EXPENSE' if expense_group.fund_source == 'PERSONAL' else 'CREATING_DEBIT_CARD_EXPENSE'}
)
Expand Down
24 changes: 16 additions & 8 deletions apps/workspaces/actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,31 +286,35 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.reimbursable_expenses_object == 'EXPENSE':
schedule_qbo_expense_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.reimbursable_expenses_object == 'CHECK':
schedule_cheques_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.reimbursable_expenses_object == 'JOURNAL ENTRY':
schedule_journal_entry_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

if general_settings.corporate_credit_card_expenses_object:
Expand All @@ -324,31 +328,35 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.corporate_credit_card_expenses_object == 'CREDIT CARD PURCHASE':
schedule_credit_card_purchase_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.corporate_credit_card_expenses_object == 'DEBIT CARD EXPENSE':
schedule_qbo_expense_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.corporate_credit_card_expenses_object == 'BILL':
schedule_bills_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)
if is_expenses_exported:
last_export_detail.last_exported_at = last_exported_at
Expand Down
Loading

0 comments on commit a83a2c1

Please sign in to comment.