Skip to content

Commit

Permalink
Merge pull request #258 from lsst-sqre/tickets/DM-38425
Browse files Browse the repository at this point in the history
DM-38425: Include the monkey and flock in Slack exceptions
rra authored May 15, 2023
2 parents 0886fc2 + 12e1eb3 commit c0b3102
Showing 8 changed files with 92 additions and 2 deletions.
4 changes: 4 additions & 0 deletions changelog.d/20230515_104223_rra_DM_38425.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### New features

- Slack alerts from monkeys now include the flock and monkey name as a field in the alert.
- Unexpected business exceptions now include an "Exception type" heading and use "Failed at" instead of "Date" to match the display of expected exceptions.
3 changes: 3 additions & 0 deletions changelog.d/20230515_112657_rra_DM_38425.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Other changes

- mobu now uses the [Ruff](https://beta.ruff.rs/docs/) linter instead of flake8, isort, and pydocstyle.
14 changes: 14 additions & 0 deletions src/mobu/exceptions.py
Original file line number Diff line number Diff line change
@@ -163,6 +163,17 @@ class MobuSlackException(SlackException):
When the operation started.
failed_at
When the operation failed (defaults to the current time).
Attributes
----------
started_at
When the operation that ended in an exception started.
monkey
The running monkey in which the exception happened.
event
Name of the business event that provoked the exception.
annotations
Additional annotations for the running business.
"""

def __init__(
@@ -175,6 +186,7 @@ def __init__(
) -> None:
super().__init__(msg, user, failed_at=failed_at)
self.started_at = started_at
self.monkey: str | None = None
self.event: str | None = None
self.annotations: dict[str, str] = {}

@@ -238,6 +250,8 @@ def common_fields(self) -> list[SlackBaseField]:
started_at = format_datetime_for_logging(self.started_at)
field = SlackTextField(heading="Started at", text=started_at)
fields.insert(0, field)
if self.monkey:
fields.append(SlackTextField(heading="Monkey", text=self.monkey))
if self.user:
fields.append(SlackTextField(heading="User", text=self.user))
if self.event:
1 change: 1 addition & 0 deletions src/mobu/services/flock.py
Original file line number Diff line number Diff line change
@@ -135,6 +135,7 @@ def _create_monkey(self, user: AuthenticatedUser) -> Monkey:
"""Create a monkey that will run as a given user."""
return Monkey(
name=user.username,
flock=self.name,
business_config=self._config.business,
user=user,
http_client=self._http_client,
22 changes: 20 additions & 2 deletions src/mobu/services/monkey.py
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
from structlog.stdlib import BoundLogger

from ..config import config
from ..exceptions import MobuSlackException
from ..models.business.base import BusinessConfig
from ..models.business.empty import EmptyLoopConfig
from ..models.business.jupyterpythonloop import JupyterPythonLoopConfig
@@ -41,6 +42,9 @@ class Monkey:
----------
name
Name of this monkey.
flock
Name of the flock this monkey belongs to, or `None` if it is running
as a solitary.
business_config
Configuration for the business it should run.
user
@@ -55,12 +59,14 @@ def __init__(
self,
*,
name: str,
flock: str | None = None,
business_config: BusinessConfig,
user: AuthenticatedUser,
http_client: AsyncClient,
logger: BoundLogger,
) -> None:
self._name = name
self._flock = flock
self._restart = business_config.restart
self._http_client = http_client
self._user = user
@@ -127,11 +133,18 @@ async def alert(self, exc: Exception) -> None:
else:
now = current_datetime(microseconds=True)
date = format_datetime_for_logging(now)
error = f"{type(exc).__name__}: {exc!s}"
name = type(exc).__name__
error = f"{name}: {exc!s}"
if self._flock:
monkey = f"{self._flock}/{self._name}"
else:
monkey = self._name
message = SlackMessage(
message=f"Unexpected exception {error}",
fields=[
SlackTextField(heading="Date", text=date),
SlackTextField(heading="Exception type", text=name),
SlackTextField(heading="Failed at", text=date),
SlackTextField(heading="Monkey", text=monkey),
SlackTextField(heading="User", text=self._user.username),
],
)
@@ -184,6 +197,11 @@ async def _runner(self) -> None:
run = False
except Exception as e:
msg = "Exception thrown while doing monkey business"
if isinstance(e, MobuSlackException):
if self._flock:
e.monkey = f"{self._flock}/{self._name}"
else:
e.monkey = self._name
self._logger.exception(msg)
await self.alert(e)
run = self._restart and self._state == MonkeyState.RUNNING
35 changes: 35 additions & 0 deletions tests/business/jupyterpythonloop_test.py
Original file line number Diff line number Diff line change
@@ -220,6 +220,11 @@ async def test_hub_failed(
"text": "*Exception type*\nJupyterWebError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser2",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser2",
@@ -306,6 +311,11 @@ async def test_redirect_loop(
"text": "*Exception type*\nJupyterWebError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
@@ -383,6 +393,11 @@ async def test_spawn_timeout(
"text": "*Exception type*\nJupyterTimeoutError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
@@ -452,6 +467,11 @@ async def test_spawn_failed(
"text": "*Exception type*\nJupyterSpawnError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
@@ -540,6 +560,11 @@ async def test_delete_timeout(
"text": "*Exception type*\nJupyterTimeoutError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
@@ -614,6 +639,11 @@ async def test_code_exception(
"text": "*Exception type*\nCodeExecutionError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
@@ -745,6 +775,11 @@ async def test_long_error(
"text": "*Exception type*\nCodeExecutionError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
5 changes: 5 additions & 0 deletions tests/business/notebookrunner_test.py
Original file line number Diff line number Diff line change
@@ -181,6 +181,11 @@ async def test_alert(
"text": "*Exception type*\nCodeExecutionError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",
10 changes: 10 additions & 0 deletions tests/business/tapqueryrunner_test.py
Original file line number Diff line number Diff line change
@@ -119,6 +119,11 @@ async def test_setup_error(
"text": "*Exception type*\nTAPClientError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/tapuser",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntapuser",
@@ -183,6 +188,11 @@ async def test_alert(
"text": "*Exception type*\nCodeExecutionError",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*Monkey*\ntest/testuser1",
"verbatim": True,
},
{
"type": "mrkdwn",
"text": "*User*\ntestuser1",

0 comments on commit c0b3102

Please sign in to comment.