Branding: improve IO resilience

bot/exts/backend/branding/_cog.py

@@ -342,14 +342,14 @@ async def synchronise(self) -> tuple[bool, bool]:
         """
         log.debug("Synchronise: fetching current event.")
 
-        current_event, available_events = await self.repository.get_current_event()
+        try:
+            current_event, available_events = await self.repository.get_current_event()
+        except Exception:
+            log.exception("Synchronisation aborted: failed to fetch events.")
+            return False, False
 
         await self.populate_cache_events(available_events)
 
-        if current_event is None:
-            log.error("Failed to fetch event. Cannot synchronise!")
-            return False, False
-
         return await self.enter_event(current_event)
 
     async def populate_cache_events(self, events: list[Event]) -> None:
@@ -433,10 +433,6 @@ async def daemon_main(self) -> None:
 
         await self.populate_cache_events(available_events)
 
-        if new_event is None:
-            log.warning("Daemon main: failed to get current event from branding repository, will do nothing.")
-            return
-
         if new_event.path != await self.cache_information.get("event_path"):
             log.debug("Daemon main: new event detected!")
             await self.enter_event(new_event)
@@ -596,10 +592,24 @@ async def branding_calendar_refresh_cmd(self, ctx: commands.Context) -> None:
         log.info("Performing command-requested event cache refresh.")
 
         async with ctx.typing():
-            available_events = await self.repository.get_events()
-            await self.populate_cache_events(available_events)
+            try:
+                available_events = await self.repository.get_events()
+            except Exception:
+                log.exception("Refresh aborted: failed to fetch events.")
+                resp = make_embed(
+                    "Refresh aborted",
+                    "Failed to fetch events. See log for details.",
+                    success=False,
+                )
+            else:
+                await self.populate_cache_events(available_events)
+                resp = make_embed(
+                    "Refresh successful",
+                    "The event calendar has been refreshed.",
+                    success=True,
+                )
 
-        await ctx.invoke(self.branding_calendar_group)
+        await ctx.send(embed=resp)
 
     # endregion
     # region: Command interface (branding daemon)

bot/exts/backend/branding/_repository.py

@@ -1,7 +1,9 @@
 import typing as t
 from datetime import UTC, date, datetime
 
+import backoff
 import frontmatter
+from aiohttp import ClientResponse
 
 from bot.bot import Bot
 from bot.constants import Keys
@@ -71,6 +73,22 @@ def __str__(self) -> str:
         return f"<Event at '{self.path}'>"
 
 
+class GitHubServerError(Exception):
+    """
+    GitHub request has failed due to a server error.
+
+    Such requests shall be retried.
+    """
+
+
+retry_on_server_error = backoff.on_exception(
+    backoff.expo,
+    GitHubServerError,
+    max_tries=5,
+    jitter=None,
+)
+
+
 class BrandingRepository:
     """
     Branding repository abstraction.
@@ -93,6 +111,18 @@ class BrandingRepository:
     def __init__(self, bot: Bot) -> None:
         self.bot = bot
 
+    def _raise_for_status(self, response: ClientResponse) -> None:
+        """
+        Raise GitHubServerError if the response status is >= 500.
+
+        The server error indicates that the request shall be retried.
+        """
+        log.trace(f"Response status: {response.status}")
+        if response.status >= 500:
+            raise GitHubServerError
+        response.raise_for_status()
+
+    @retry_on_server_error
     async def fetch_directory(self, path: str, types: t.Container[str] = ("file", "dir")) -> dict[str, RemoteObject]:
         """
         Fetch directory found at `path` in the branding repository.
@@ -105,14 +135,12 @@ async def fetch_directory(self, path: str, types: t.Container[str] = ("file", "d
         log.debug(f"Fetching directory from branding repository: '{full_url}'.")
 
         async with self.bot.http_session.get(full_url, params=PARAMS, headers=HEADERS) as response:
-            if response.status != 200:
-                raise RuntimeError(f"Failed to fetch directory due to status: {response.status}")
-
-            log.debug("Fetch successful, reading JSON response.")
+            self._raise_for_status(response)
             json_directory = await response.json()
 
         return {file["name"]: RemoteObject(file) for file in json_directory if file["type"] in types}
 
+    @retry_on_server_error
     async def fetch_file(self, download_url: str) -> bytes:
         """
         Fetch file as bytes from `download_url`.
@@ -122,10 +150,7 @@ async def fetch_file(self, download_url: str) -> bytes:
         log.debug(f"Fetching file from branding repository: '{download_url}'.")
 
         async with self.bot.http_session.get(download_url, params=PARAMS, headers=HEADERS) as response:
-            if response.status != 200:
-                raise RuntimeError(f"Failed to fetch file due to status: {response.status}")
-
-            log.debug("Fetch successful, reading payload.")
+            self._raise_for_status(response)
             return await response.read()
 
     def parse_meta_file(self, raw_file: bytes) -> MetaFile:
@@ -186,37 +211,30 @@ async def get_events(self) -> list[Event]:
         """
         Discover available events in the branding repository.
 
-        Misconfigured events are skipped. May return an empty list in the catastrophic case.
+        Propagate errors if an event fails to fetch or deserialize. Such errors cannot be handled
+        and the operation should be retried in the future.
         """
         log.debug("Discovering events in branding repository.")
 
-        try:
-            event_directories = await self.fetch_directory("events", types=("dir",))  # Skip files.
-        except Exception:
-            log.exception("Failed to fetch 'events' directory.")
-            return []
+        event_directories = await self.fetch_directory("events", types=("dir",))  # Skip files.
 
         instances: list[Event] = []
 
         for event_directory in event_directories.values():
-            log.trace(f"Attempting to construct event from directory: '{event_directory.path}'.")
-            try:
-                instance = await self.construct_event(event_directory)
-            except Exception as exc:
-                log.warning(f"Could not construct event '{event_directory.path}'.", exc_info=exc)
-            else:
-                instances.append(instance)
+            log.trace(f"Reading event directory: '{event_directory.path}'.")
+            instance = await self.construct_event(event_directory)
+            instances.append(instance)
 
         return instances
 
-    async def get_current_event(self) -> tuple[Event | None, list[Event]]:
+    async def get_current_event(self) -> tuple[Event, list[Event]]:
         """
         Get the currently active event, or the fallback event.
 
         The second return value is a list of all available events. The caller may discard it, if not needed.
         Returning all events alongside the current one prevents having to query the API twice in some cases.
 
-        The current event may be None in the case that no event is active, and no fallback event is found.
+        If no event is active and the fallback event cannot be found, raise an error.
         """
         utc_now = datetime.now(tz=UTC)
         log.debug(f"Finding active event for: {utc_now}.")
@@ -239,5 +257,4 @@ async def get_current_event(self) -> tuple[Event | None, list[Event]]:
             if event.meta.is_fallback:
                 return event, available_events
 
-        log.warning("No event is currently active and no fallback event was found!")
-        return None, available_events
+        raise RuntimeError("No event is currently active and no fallback event was found!")

pyproject.toml

@@ -13,6 +13,7 @@ pydis_core = { version = "10.5.1", extras = ["async-rediscache"] }
 
 aiohttp = "3.9.1"
 arrow = "1.3.0"
+backoff = "2.2.1"
 beautifulsoup4 = "4.12.2"
 colorama = { version = "0.4.6", markers = "sys_platform == 'win32'" }
 coloredlogs = "15.0.1"