diff --git a/data_integration/config.py b/data_integration/config.py index 5d8dd8c..f888de7 100644 --- a/data_integration/config.py +++ b/data_integration/config.py @@ -24,6 +24,11 @@ def default_db_alias() -> str: return 'dwh-etl' +def default_task_max_retries(): + """How many times a task is retried when it fails by default """ + return 0 + + def first_date() -> datetime.date: """Ignore data before this date""" return datetime.date(2000, 1, 1) diff --git a/data_integration/execution.py b/data_integration/execution.py index 2eabc28..94be0b8 100644 --- a/data_integration/execution.py +++ b/data_integration/execution.py @@ -373,10 +373,11 @@ def run(self): try: while True: if not self.task.run(): - if attempt < self.task.max_retries: + max_retries = self.task.max_retries or config.default_task_max_retries() + if attempt < max_retries: attempt += 1 delay = pow(2, attempt + 2) - logger.log(message=f'Retry {attempt}/{self.task.max_retries} in {delay} seconds', + logger.log(message=f'Retry {attempt}/{max_retries} in {delay} seconds', is_error=True, format=logger.Format.ITALICS) time.sleep(delay) else: diff --git a/data_integration/pipelines.py b/data_integration/pipelines.py index 287400e..4f34c3c 100644 --- a/data_integration/pipelines.py +++ b/data_integration/pipelines.py @@ -81,7 +81,7 @@ def html_doc_items(self) -> [(str, str)]: class Task(Node): - def __init__(self, id: str, description: str, commands: [Command] = None, max_retries: int = 0) -> None: + def __init__(self, id: str, description: str, commands: [Command] = None, max_retries: int = None) -> None: super().__init__(id, description) self.commands = [] self.max_retries = max_retries