Skip to content

Commit

Permalink
fix context precision conflicts with prerelease
Browse files Browse the repository at this point in the history
  • Loading branch information
kcortinas authored May 24, 2024
1 parent ca562a1 commit 12a813b
Showing 1 changed file with 24 additions and 6 deletions.
30 changes: 24 additions & 6 deletions rag_experiment_accelerator/evaluation/eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,19 @@ def remove_spaces(text):
return text.strip()


def lower_and_strip(text):
"""
Converts the input to lowercase without spaces or empty string if None.
Args:
text (str): The string to format.
Returns:
str: The formatted input string.
"""
if text is None return '' else return text.lower().strip()


# https://huggingface.co/spaces/evaluate-metric/bleu
def bleu(predictions, references):
bleu = evaluate.load("bleu")
Expand Down Expand Up @@ -313,17 +326,22 @@ def llm_context_precision(
context=context,
question=question,
)
if result is None:
logger.warning("Unable to generate context precision score")

llm_judge_response = lower_and_strip(result)
# Since we're only asking for one response, the result is always a boolean 1 or 0
if result.lower().strip() == "yes":
if llm_judge_response == "yes":
relevancy_scores.append(1)
elif result.lower().strip() == "no":
elif llm_judge_response == "no":
relevancy_scores.append(0)
else:
logger.warning("Unable to generate context precision score")

logger.debug(relevancy_scores)
return (sum(relevancy_scores) / len(relevancy_scores)) * 100

if not relevancy_scores:
logger.warning("Unable to compute average context precision")
return -1
else:
return (sum(relevancy_scores) / len(relevancy_scores)) * 100


def llm_context_recall(
Expand Down

0 comments on commit 12a813b

Please sign in to comment.