mark task as failued if ADO notification fails

src/api-service/__app__/onefuzzlib/notifications/ado.py

@@ -19,8 +19,10 @@
     WorkItemTrackingClient,
 )
 from memoization import cached
-from onefuzztypes.models import ADOTemplate, Report
+from onefuzztypes.enums import ErrorCode, TaskState
+from onefuzztypes.models import ADOTemplate, Error, Report
 
+from ..tasks.main import Task
 from .common import Render
 
 
@@ -194,13 +196,36 @@ def process(self) -> None:
             self.create_new()
 
 
+def fail_task(report: Report, error: Exception) -> None:
+    logging.error(
+        "ADO report failed: job_id:%s task_id:%s err:%s",
+        report.job_id,
+        report.task_id,
+        error,
+    )
+
+    task = Task.get(report.job_id, report.task_id)
+    if task:
+        task.error = Error(code=ErrorCode.NOTIFICATION_FAILURE, errors=[str(error)])
+        task.state = TaskState.stopping
+        task.save()
+
+
 def notify_ado(
     config: ADOTemplate, container: str, filename: str, report: Report
 ) -> None:
+    logging.info(
+        "notify ado: job_id:%s task_id:%s container:%s filename:%s",
+        report.job_id,
+        report.task_id,
+        container,
+        filename,
+    )
+
     try:
         ado = ADO(container, filename, config, report)
         ado.process()
     except AzureDevOpsServiceError as err:
-        logging.error("ADO report failed: %s", err)
+        fail_task(report, err)
     except ValueError as err:
-        logging.error("ADO report value error: %s", err)
+        fail_task(report, err)

src/pytypes/onefuzztypes/enums.py

@@ -227,6 +227,7 @@ class ErrorCode(Enum):
     UNABLE_TO_FIND = 467
     TASK_FAILED = 468
     INVALID_NODE = 469
+    NOTIFICATION_FAILURE = 470
 
 
 class HeartbeatType(Enum):