add config default_task_max_retries

data_integration/config.py

@@ -24,6 +24,11 @@ def default_db_alias() -> str:
     return 'dwh-etl'
 
 
+def default_task_max_retries():
+    """How many times a task is retried when it fails by default """
+    return 0
+
+
 def first_date() -> datetime.date:
     """Ignore data before this date"""
     return datetime.date(2000, 1, 1)

data_integration/execution.py

@@ -373,10 +373,11 @@ def run(self):
         try:
             while True:
                 if not self.task.run():
-                    if attempt < self.task.max_retries:
+                    max_retries = self.task.max_retries or config.default_task_max_retries()
+                    if attempt < max_retries:
                         attempt += 1
                         delay = pow(2, attempt + 2)
-                        logger.log(message=f'Retry {attempt}/{self.task.max_retries} in {delay} seconds',
+                        logger.log(message=f'Retry {attempt}/{max_retries} in {delay} seconds',
                                    is_error=True, format=logger.Format.ITALICS)
                         time.sleep(delay)
                     else:

data_integration/pipelines.py

@@ -81,7 +81,7 @@ def html_doc_items(self) -> [(str, str)]:
 
 
 class Task(Node):
-    def __init__(self, id: str, description: str, commands: [Command] = None, max_retries: int = 0) -> None:
+    def __init__(self, id: str, description: str, commands: [Command] = None, max_retries: int = None) -> None:
         super().__init__(id, description)
         self.commands = []
         self.max_retries = max_retries