feat: Evals with explanations

src/phoenix/experimental/evals/__init__.py

@@ -2,15 +2,16 @@
 from .models import OpenAIModel, VertexAIModel
 from .retrievals import compute_precisions_at_k
 from .templates import (
-    CODE_READABILITY_PROMPT_RAILS_MAP,
-    CODE_READABILITY_PROMPT_TEMPLATE_STR,
-    HALLUCINATION_PROMPT_RAILS_MAP,
-    HALLUCINATION_PROMPT_TEMPLATE_STR,
+    CODE_READABILITY_PROMPT_RAILS,
+    CODE_READABILITY_PROMPT_TEMPLATE,
+    HALLUCINATION_PROMPT_RAILS,
+    HALLUCINATION_PROMPT_TEMPLATE,
     NOT_PARSABLE,
-    RAG_RELEVANCY_PROMPT_RAILS_MAP,
-    RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
-    TOXICITY_PROMPT_RAILS_MAP,
-    TOXICITY_PROMPT_TEMPLATE_STR,
+    RAG_RELEVANCY_PROMPT_RAILS,
+    RAG_RELEVANCY_PROMPT_TEMPLATE,
+    TOXICITY_PROMPT_RAILS,
+    TOXICITY_PROMPT_TEMPLATE,
+    ClassificationTemplate,
     PromptTemplate,
 )
 from .utils.downloads import download_benchmark_dataset
@@ -23,14 +24,15 @@
     "OpenAIModel",
     "VertexAIModel",
     "PromptTemplate",
-    "CODE_READABILITY_PROMPT_RAILS_MAP",
-    "CODE_READABILITY_PROMPT_TEMPLATE_STR",
-    "HALLUCINATION_PROMPT_RAILS_MAP",
-    "HALLUCINATION_PROMPT_TEMPLATE_STR",
-    "RAG_RELEVANCY_PROMPT_RAILS_MAP",
-    "RAG_RELEVANCY_PROMPT_TEMPLATE_STR",
-    "TOXICITY_PROMPT_TEMPLATE_STR",
-    "TOXICITY_PROMPT_RAILS_MAP",
+    "ClassificationTemplate",
+    "CODE_READABILITY_PROMPT_RAILS",
+    "CODE_READABILITY_PROMPT_TEMPLATE",
+    "HALLUCINATION_PROMPT_RAILS",
+    "HALLUCINATION_PROMPT_TEMPLATE",
+    "RAG_RELEVANCY_PROMPT_RAILS",
+    "RAG_RELEVANCY_PROMPT_TEMPLATE",
+    "TOXICITY_PROMPT_RAILS",
+    "TOXICITY_PROMPT_TEMPLATE",
     "NOT_PARSABLE",
     "run_relevance_eval",
 ]

src/phoenix/experimental/evals/functions/classify.py

@@ -8,11 +8,12 @@
 from phoenix.experimental.evals.models import BaseEvalModel, OpenAIModel, set_verbosity
 from phoenix.experimental.evals.templates import (
     NOT_PARSABLE,
-    RAG_RELEVANCY_PROMPT_RAILS_MAP,
-    RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
-    PromptTemplate,
+    RAG_RELEVANCY_PROMPT_RAILS,
+    RAG_RELEVANCY_PROMPT_TEMPLATE,
+    ClassificationTemplate,
+    PromptOptions,
     map_template,
-    normalize_template,
+    normalize_classification_template,
 )
 from phoenix.trace.semantic_conventions import DOCUMENT_CONTENT, INPUT_VALUE, RETRIEVAL_DOCUMENTS
 from phoenix.utilities.logging import printif
@@ -32,7 +33,7 @@
 def llm_classify(
     dataframe: pd.DataFrame,
     model: BaseEvalModel,
-    template: Union[PromptTemplate, str],
+    template: Union[ClassificationTemplate, str],
     rails: List[str],
     system_instruction: Optional[str] = None,
     verbose: bool = False,
@@ -85,31 +86,36 @@ def llm_classify(
         and model.supports_function_calling
     )
 
-    # TODO: support explanation without function calling
-    if provide_explanation and not use_openai_function_call:
-        raise ValueError(
-            "explanation is not currently available for models without OpenAI function calling"
-        )
-
     model_kwargs: Dict[str, Any] = {}
     if use_openai_function_call:
         openai_function = _default_openai_function(rails, provide_explanation)
         model_kwargs["functions"] = [openai_function]
         model_kwargs["function_call"] = {"name": openai_function["name"]}
 
-    eval_template = normalize_template(template)
-    prompts = map_template(dataframe, eval_template)
+    eval_template = normalize_classification_template(rails=rails, template=template)
+
+    prompt_options = PromptOptions(provide_explanation=provide_explanation)
+    prompts = map_template(dataframe, eval_template, options=prompt_options)
+
     labels: List[str] = []
     explanations: List[Optional[str]] = []
+
     if generation_info := model.verbose_generation_info():
         printif(verbose, generation_info)
 
     for prompt in tqdm(prompts):
         with set_verbosity(model, verbose) as verbose_model:
             response = verbose_model(prompt, instruction=system_instruction, **model_kwargs)
         if not use_openai_function_call:
-            unrailed_label = response
-            explanation = None
+            if provide_explanation:
+                unrailed_label, explanation = eval_template.parse_label(response), response
+                printif(
+                    verbose and unrailed_label == NOT_PARSABLE,
+                    f"- Could not parse {repr(response)}",
+                )
+            else:
+                unrailed_label = response
+                explanation = None
         else:
             try:
                 function_arguments = json.loads(response, strict=False)
@@ -132,8 +138,8 @@ def llm_classify(
 def run_relevance_eval(
     dataframe: pd.DataFrame,
     model: BaseEvalModel,
-    template: Union[PromptTemplate, str] = RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
-    rails: List[str] = list(RAG_RELEVANCY_PROMPT_RAILS_MAP.values()),
+    template: Union[ClassificationTemplate, str] = RAG_RELEVANCY_PROMPT_TEMPLATE,
+    rails: List[str] = list(RAG_RELEVANCY_PROMPT_RAILS),
     system_instruction: Optional[str] = None,
     query_column_name: str = "query",
     document_column_name: str = "reference",

src/phoenix/experimental/evals/functions/generate.py

@@ -4,7 +4,11 @@
 import pandas as pd
 
 from phoenix.experimental.evals.models import BaseEvalModel, set_verbosity
-from phoenix.experimental.evals.templates import PromptTemplate, map_template, normalize_template
+from phoenix.experimental.evals.templates import (
+    PromptTemplate,
+    map_template,
+    normalize_prompt_template,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +48,8 @@ def llm_generate(
 
     """
     with set_verbosity(model, verbose) as verbose_model:
-        template = normalize_template(template)
-        logger.info(f"Template: \n{template.text}\n")
+        template = normalize_prompt_template(template)
+        logger.info(f"Template: \n{template.prompt()}\n")
         logger.info(f"Template variables: {template.variables}")
         prompts = map_template(dataframe, template)
 

src/phoenix/experimental/evals/templates/__init__.py

@@ -1,26 +1,38 @@
 from .default_templates import (
-    CODE_READABILITY_PROMPT_RAILS_MAP,
-    CODE_READABILITY_PROMPT_TEMPLATE_STR,
-    HALLUCINATION_PROMPT_RAILS_MAP,
-    HALLUCINATION_PROMPT_TEMPLATE_STR,
-    RAG_RELEVANCY_PROMPT_RAILS_MAP,
-    RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
-    TOXICITY_PROMPT_RAILS_MAP,
-    TOXICITY_PROMPT_TEMPLATE_STR,
+    CODE_READABILITY_PROMPT_RAILS,
+    CODE_READABILITY_PROMPT_TEMPLATE,
+    HALLUCINATION_PROMPT_RAILS,
+    HALLUCINATION_PROMPT_TEMPLATE,
+    RAG_RELEVANCY_PROMPT_RAILS,
+    RAG_RELEVANCY_PROMPT_TEMPLATE,
+    TOXICITY_PROMPT_RAILS,
+    TOXICITY_PROMPT_TEMPLATE,
+)
+from .template import (
+    NOT_PARSABLE,
+    ClassificationTemplate,
+    PromptOptions,
+    PromptTemplate,
+    map_template,
+    normalize_classification_template,
+    normalize_prompt_template,
 )
-from .template import NOT_PARSABLE, PromptTemplate, map_template, normalize_template
 
 __all__ = [
+    "UserTemplate",
+    "PromptOptions",
     "PromptTemplate",
-    "normalize_template",
+    "ClassificationTemplate",
+    "normalize_classification_template",
+    "normalize_prompt_template",
     "map_template",
     "NOT_PARSABLE",
-    "RAG_RELEVANCY_PROMPT_RAILS_MAP",
-    "RAG_RELEVANCY_PROMPT_TEMPLATE_STR",
-    "HALLUCINATION_PROMPT_RAILS_MAP",
-    "HALLUCINATION_PROMPT_TEMPLATE_STR",
-    "CODE_READABILITY_PROMPT_RAILS_MAP",
-    "CODE_READABILITY_PROMPT_TEMPLATE_STR",
-    "TOXICITY_PROMPT_RAILS_MAP",
-    "TOXICITY_PROMPT_TEMPLATE_STR",
+    "CODE_READABILITY_PROMPT_RAILS",
+    "CODE_READABILITY_PROMPT_TEMPLATE",
+    "HALLUCINATION_PROMPT_RAILS",
+    "HALLUCINATION_PROMPT_TEMPLATE",
+    "RAG_RELEVANCY_PROMPT_RAILS",
+    "RAG_RELEVANCY_PROMPT_TEMPLATE",
+    "TOXICITY_PROMPT_RAILS",
+    "TOXICITY_PROMPT_TEMPLATE",
 ]