Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Better ErrorEvents #155

Closed
wants to merge 6 commits into from
+25 −22
Closed

[Feature] Better ErrorEvents #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4160cd2
add error types
siyangqiu Apr 19, 2024
f2fa53e
automatically capture stacktrace
siyangqiu Apr 19, 2024
8de3b65
fix issues
siyangqiu Apr 19, 2024
50a5b3e
fix traceback
siyangqiu Apr 19, 2024
6843d37
actual traceback fix
siyangqiu Apr 19, 2024
86d8cdc
remove stack trace todos
siyangqiu Apr 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions agentops/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@ def _record_event_sync(self, func, event_name, *args, **kwargs):
self.record(event)

except Exception as e:
# TODO: add the stack trace
self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
self.record(ErrorEvent(trigger_event=event, error_type=type(e).__name__, details=str(e)))

# Re-raise the exception
raise
Expand Down Expand Up @@ -190,8 +189,7 @@ async def _record_event_async(self, func, event_name, *args, **kwargs):
self.record(event)

except Exception as e:
# TODO: add the stack trace
self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
self.record(ErrorEvent(trigger_event=event, error_type=type(e).__name__, details=str(e)))

# Re-raise the exception
raise
Expand Down
4 changes: 2 additions & 2 deletions agentops/event.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from dataclasses import dataclass, field
from typing import List, Optional
from .helpers import get_ISO_time, check_call_stack_for_agent_id
from .helpers import get_ISO_time, get_traceback, check_call_stack_for_agent_id
from .enums import EventType, Models
from uuid import UUID, uuid4

Expand Down Expand Up @@ -127,7 +127,7 @@ class ErrorEvent():
error_type: Optional[str] = None
code: Optional[str] = None
details: Optional[str] = None
logs: Optional[str] = None
logs: Optional[str] = field(default_factory=get_traceback)
timestamp: str = field(default_factory=get_ISO_time)

def __post_init__(self):
Expand Down
5 changes: 5 additions & 0 deletions agentops/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import json
import inspect
import logging
import traceback
from uuid import UUID
import os
from importlib.metadata import version
Expand All @@ -30,6 +31,10 @@ def get_ISO_time():
"""
return datetime.utcfromtimestamp(time.time()).isoformat(timespec='milliseconds') + 'Z'

def get_traceback():
"""Returns the current traceback as a string."""
return traceback.format_exc()


def is_jsonable(x):
try:
Expand Down
24 changes: 12 additions & 12 deletions agentops/langchain_callback_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def on_llm_error(
llm_event: LLMEvent = self.events.llm[str(run_id)]
self.ao_client.record(llm_event)

error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=llm_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
Copy link
Contributor

@bboynton97 bboynton97 Apr 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking more like

error_event = ErrorEvent(trigger_event=llm_event, error=error)

and then the ErrorEvent constructor does the rest:

def __init__(self, trigger_event, error):
  self.error_type = error_type=type(error).__name__
  self.details = str(error)
  self.timestamp = get_ISO_time()

it's less work for the developer to have to do. more magic

Copy link
Contributor

@bboynton97 bboynton97 Apr 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or actually maybe like this

def __init__(self, trigger_event, error, error_type = None, details = None, timestamp = None):
  self.error_type = error_type or error_type=type(error).__name__
  self.details =  details or str(error)
  self.timestamp = timestamp or get_ISO_time()

self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -106,7 +106,7 @@ def on_llm_end(

if len(response.generations) == 0:
# TODO: more descriptive error
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations",
details="on_llm_end: No generations", timestamp=get_ISO_time())
self.ao_client.record(error_event)

Expand Down Expand Up @@ -156,7 +156,7 @@ def on_chain_error(
action_event: ActionEvent = self.events.chain[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -199,7 +199,7 @@ def on_tool_end(
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
if kwargs.get('name') == '_Exception':
error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output, timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -214,7 +214,7 @@ def on_tool_error(
tool_event: ToolEvent = self.events.tool[str(run_id)]
self.ao_client.record(tool_event)

error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -265,7 +265,7 @@ def on_retriever_error(
action_event: ActionEvent = self.events.retriever[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -405,7 +405,7 @@ async def on_llm_error(
llm_event: LLMEvent = self.events.llm[str(run_id)]
self.ao_client.record(llm_event)

error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=llm_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -431,7 +431,7 @@ async def on_llm_end(

if len(response.generations) == 0:
# TODO: more descriptive error
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations",
details="on_llm_end: No generations", timestamp=get_ISO_time())
self.ao_client.record(error_event)

Expand Down Expand Up @@ -481,7 +481,7 @@ async def on_chain_error(
action_event: ActionEvent = self.events.chain[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -524,7 +524,7 @@ async def on_tool_end(
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
if kwargs.get('name') == '_Exception':
error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output, timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -539,7 +539,7 @@ async def on_tool_error(
tool_event: ToolEvent = self.events.tool[str(run_id)]
self.ao_client.record(tool_event)

error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -590,7 +590,7 @@ async def on_retriever_error(
action_event: ActionEvent = self.events.retriever[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, error_type=type(error).__name__, details=str(error), timestamp=get_ISO_time())
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down
8 changes: 4 additions & 4 deletions agentops/llm_tracker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def handle_stream_chunk(chunk):

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, error_type=type(e).__name__, details=str(e)))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down Expand Up @@ -97,7 +97,7 @@ def generator():

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, error_type=type(e).__name__, details=str(e)))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down Expand Up @@ -143,7 +143,7 @@ def handle_stream_chunk(chunk: ChatCompletionChunk):

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, error_type=type(e).__name__, details=str(e)))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down Expand Up @@ -188,7 +188,7 @@ async def async_generator():

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, error_type=type(e).__name__, details=str(e)))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down
Loading