From 546c850a43d8b00fafc11e02e63fa5caa56b4c07 Mon Sep 17 00:00:00 2001 From: Krzysztof Furman Date: Fri, 13 Oct 2023 13:05:53 +0100 Subject: [PATCH] Error handling for when Azure container log cannot be read in properly. (#34627) * Error handling for broken or null Azure logs. * Updated logging syntax to match rest of the code. * Moved broken log checking closer to the source to avoid catching more general errors. * Fixed style issues. * Changed log exception to log error. * Changed failure case return type to list. Not an empty list as that ruins normal operation - instead [None. * Cleaner checking of logs list not being empty Co-authored-by: Wei Lee --------- Co-authored-by: Wei Lee --- airflow/providers/microsoft/azure/hooks/container_instance.py | 2 ++ .../providers/microsoft/azure/operators/container_instances.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/airflow/providers/microsoft/azure/hooks/container_instance.py b/airflow/providers/microsoft/azure/hooks/container_instance.py index 34edbf74bb673..c8fa67ca59a7a 100644 --- a/airflow/providers/microsoft/azure/hooks/container_instance.py +++ b/airflow/providers/microsoft/azure/hooks/container_instance.py @@ -169,6 +169,8 @@ def get_logs(self, resource_group: str, name: str, tail: int = 1000) -> list: :return: A list of log messages """ logs = self.connection.containers.list_logs(resource_group, name, name, tail=tail) + if logs.content is None: + return [None] return logs.content.splitlines(True) def delete(self, resource_group: str, name: str) -> None: diff --git a/airflow/providers/microsoft/azure/operators/container_instances.py b/airflow/providers/microsoft/azure/operators/container_instances.py index 8abcb48c13662..30b4877d3efc0 100644 --- a/airflow/providers/microsoft/azure/operators/container_instances.py +++ b/airflow/providers/microsoft/azure/operators/container_instances.py @@ -318,6 +318,9 @@ def _monitor_logging(self, resource_group: str, name: str) -> int: if state in ["Running", "Terminated", "Succeeded"]: try: logs = self._ci_hook.get_logs(resource_group, name) + if logs and logs[0] is None: + self.log.error("Container log is broken, marking as failed.") + return 1 last_line_logged = self._log_last(logs, last_line_logged) except CloudError: self.log.exception(