Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update context relevancy to rely on uptrains parse json function to solve parsing error. Added more informative error logging. #742

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Update context relevancy to rely on uptrains parse json function to solve parsing error. Added more informative error logging. #742

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions uptrain/operators/language/context_quality.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from uptrain.framework import Settings
from uptrain.operators.base import register_op, ColumnOp, TYPE_TABLE_OUTPUT
from uptrain.utilities import polars_to_json_serializable_dict
from uptrain.operators.language.llm import LLMMulticlient
from uptrain.operators.language.llm import LLMMulticlient, parse_json

from uptrain.operators.language.prompts.classic import (
CONTEXT_CONCISENESS_PROMPT_TEMPLATE,
Expand Down Expand Up @@ -183,23 +183,27 @@ def evaluate_local(self, data):
"explanation_context_relevance": None,
}
try:
score = self.score_mapping[
json.loads(res.response.choices[0].message.content)["Choice"]
]
content = res.response.choices[0].message.content
parsed_content = parse_json(content)
#parsed_content = json.loads(content)
choice = parsed_content["Choice"]
score = self.score_mapping[choice]
output["score_context_relevance"] = float(score)
output["explanation_context_relevance"] = res.response.choices[
0
].message.content
except Exception:
logger.error(
f"Error when processing payload at index {idx}: {res.error}"
)
output["explanation_context_relevance"] = content
except json.JSONDecodeError as e:
logger.error(f"JSONDecodeError at index {idx}: {str(e)}")
logger.error(f"Printing res: {res}")
logger.error(f"Raw content: {content}")
except KeyError as e:
logger.error(f"KeyError at index {idx}: {str(e)}")
logger.error(f"Parsed content: {parsed_content}")
except Exception as e:
logger.error(f"Unexpected error at index {idx}: {type(e).__name__}: {str(e)}")
results.append((idx, output))
results = [val for _, val in sorted(results, key=lambda x: x[0])]

return results


@register_op
class ResponseCompletenessWrtContext(ColumnOp):
"""
Expand Down