Skip to content

Commit a5508e7

Browse files
zszabo-rhzszabo-rh
committed
MGMT-21204: add categories to feedback API
1 parent 80257e0 commit a5508e7

File tree

5 files changed

+346
-35
lines changed

5 files changed

+346
-35
lines changed

‎docs/openapi.json‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -935,6 +935,37 @@
935935
"title": "DataCollectorConfiguration",
936936
"description": "Data collector configuration for sending data to ingress server."
937937
},
938+
"FeedbackCategory": {
939+
"type": "string",
940+
"enum": [
941+
"incorrect",
942+
"hallucinated_content",
943+
"not_relevant",
944+
"missing_context",
945+
"outdated_information",
946+
"incomplete",
947+
"too_brief",
948+
"too_verbose",
949+
"unclear",
950+
"unsafe",
951+
"biased"
952+
],
953+
"x-enum-varnames": [
954+
"INCORRECT",
955+
"HALLUCINATED_CONTENT",
956+
"NOT_RELEVANT",
957+
"MISSING_CONTEXT",
958+
"OUTDATED_INFORMATION",
959+
"INCOMPLETE",
960+
"TOO_BRIEF",
961+
"TOO_VERBOSE",
962+
"UNCLEAR",
963+
"UNSAFE",
964+
"BIASED"
965+
],
966+
"title": "FeedbackCategory",
967+
"description": "Enum representing predefined feedback categories for AI responses.\n\nThese categories help provide structured feedback about AI inference quality.\nMultiple categories can be selected to provide comprehensive feedback."
968+
},
938969
"FeedbackRequest": {
939970
"properties": {
940971
"conversation_id": {
@@ -975,6 +1006,27 @@
9751006
"examples": [
9761007
"I'm not satisfied with the response because it is too vague."
9771008
]
1009+
},
1010+
"categories": {
1011+
"anyOf": [
1012+
{
1013+
"items": {
1014+
"$ref": "#/components/schemas/FeedbackCategory"
1015+
},
1016+
"type": "array"
1017+
},
1018+
{
1019+
"type": "null"
1020+
}
1021+
],
1022+
"title": "Categories",
1023+
"description": "List of feedback categories that apply to the LLM response.",
1024+
"examples": [
1025+
[
1026+
"too_brief",
1027+
"unclear"
1028+
]
1029+
]
9781030
}
9791031
},
9801032
"type": "object",
@@ -984,14 +1036,26 @@
9841036
"llm_response"
9851037
],
9861038
"title": "FeedbackRequest",
987-
"description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n\nExample:\n ```python\n feedback_request = FeedbackRequest(\n conversation_id=\"12345678-abcd-0000-0123-456789abcdef\",\n user_question=\"what are you doing?\",\n user_feedback=\"Great service!\",\n llm_response=\"I don't know\",\n sentiment=-1,\n )\n ```",
1039+
"description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of feedback categories (multi-select).\n\nExample:\n ```python\n feedback_request = FeedbackRequest(\n conversation_id=\"12345678-abcd-0000-0123-456789abcdef\",\n user_question=\"what are you doing?\",\n user_feedback=\"Great service!\",\n llm_response=\"I don't know\",\n sentiment=-1,\n categories=[FeedbackCategory.INCOMPLETE, FeedbackCategory.UNSAFE]\n )\n ```",
9881040
"examples": [
9891041
{
9901042
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
9911043
"llm_response": "bar",
9921044
"sentiment": 1,
9931045
"user_feedback": "Great service!",
9941046
"user_question": "foo"
1047+
},
1048+
{
1049+
"categories": [
1050+
"too_brief",
1051+
"incomplete",
1052+
"missing_context"
1053+
],
1054+
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
1055+
"llm_response": "You need to use Docker and Kubernetes for everything.",
1056+
"sentiment": -1,
1057+
"user_feedback": "This response is too general and doesn't provide specific steps.",
1058+
"user_question": "How do I deploy a web app?"
9951059
}
9961060
]
9971061
},

‎docs/openapi.md‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,29 @@ Data collector configuration for sending data to ingress server.
457457
| connection_timeout | integer | |
458458

459459

460+
## FeedbackCategory
461+
462+
463+
Enum representing predefined feedback categories for AI responses.
464+
465+
These categories help provide structured feedback about AI inference quality.
466+
Multiple categories can be selected to provide comprehensive feedback.
467+
468+
| Value | Description |
469+
|-------|-------------|
470+
| incorrect | The answer provided is completely wrong |
471+
| hallucinated_content | The response includes made-up information that doesn't exist |
472+
| not_relevant | This answer doesn't address my question at all |
473+
| missing_context | The response lacks important context needed to understand the topic |
474+
| outdated_information | This information is from several years ago and no longer accurate |
475+
| incomplete | The answer only covers part of what I asked about |
476+
| too_brief | This response is too short and doesn't provide enough detail |
477+
| too_verbose | The answer is unnecessarily long and includes too much irrelevant information |
478+
| unclear | The explanation is confusing and hard to understand |
479+
| unsafe | This response could be harmful or dangerous if followed |
480+
| biased | The answer shows clear bias against certain groups or viewpoints |
481+
482+
460483
## FeedbackRequest
461484

462485

@@ -468,15 +491,27 @@ Attributes:
468491
llm_response: The required LLM response.
469492
sentiment: The optional sentiment.
470493
user_feedback: The optional user feedback.
494+
categories: The optional list of feedback categories (multi-select).
471495

472-
Example:
496+
Examples:
473497
```python
498+
# Basic feedback
474499
feedback_request = FeedbackRequest(
475500
conversation_id="12345678-abcd-0000-0123-456789abcdef",
476501
user_question="what are you doing?",
477502
user_feedback="Great service!",
478503
llm_response="I don't know",
504+
sentiment=1
505+
)
506+
507+
# Feedback with categories
508+
feedback_request = FeedbackRequest(
509+
conversation_id="12345678-abcd-0000-0123-456789abcdef",
510+
user_question="How do I deploy a web app?",
511+
llm_response="You need to use Docker and Kubernetes for everything.",
512+
user_feedback="This response is too general and doesn't provide specific steps.",
479513
sentiment=-1,
514+
categories=["too_brief", "incomplete", "missing_context"]
480515
)
481516
```
482517

@@ -488,6 +523,7 @@ Example:
488523
| llm_response | string | |
489524
| sentiment | | |
490525
| user_feedback | | Feedback on the LLM response. |
526+
| categories | array[FeedbackCategory] | List of feedback categories that apply to the LLM response. |
491527

492528

493529
## FeedbackResponse

‎src/models/requests.py‎

Lines changed: 83 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Model for service requests."""
22

33
from typing import Optional, Self
4+
from enum import Enum
45

56
from pydantic import BaseModel, model_validator, field_validator, Field
67
from llama_stack_client.types.agents.turn_create_params import Document
@@ -146,6 +147,36 @@ def validate_media_type(self) -> Self:
146147
return self
147148

148149

150+
class FeedbackCategory(str, Enum):
151+
"""Enum representing predefined feedback categories for AI responses.
152+
153+
These categories help provide structured feedback about AI inference quality
154+
when users provide negative feedback (thumbs down). Multiple categories can
155+
be selected to provide comprehensive feedback about response issues.
156+
"""
157+
158+
# Quality issues
159+
INCORRECT = "incorrect" # "The answer provided is completely wrong"
160+
HALLUCINATED_CONTENT = "hallucinated_content" # "The response includes made-up information that doesn't exist"
161+
162+
# Relevance and context issues
163+
NOT_RELEVANT = "not_relevant" # "This answer doesn't address my question at all"
164+
MISSING_CONTEXT = "missing_context" # "The response lacks important context needed to understand the topic"
165+
OUTDATED_INFORMATION = "outdated_information" # "This information is from several years ago and no longer accurate"
166+
167+
# Completeness issues
168+
INCOMPLETE = "incomplete" # "The answer only covers part of what I asked about"
169+
TOO_BRIEF = "too_brief" # "This response is too short and doesn't provide enough detail"
170+
TOO_VERBOSE = "too_verbose" # "The answer is unnecessarily long and includes too much irrelevant information"
171+
172+
# Clarity issues
173+
UNCLEAR = "unclear" # "The explanation is confusing and hard to understand"
174+
175+
# Safety and bias issues
176+
UNSAFE = "unsafe" # "This response could be harmful or dangerous if followed"
177+
BIASED = "biased" # "The answer shows clear bias against certain groups or viewpoints"
178+
179+
149180
class FeedbackRequest(BaseModel):
150181
"""Model representing a feedback request.
151182
@@ -155,15 +186,17 @@ class FeedbackRequest(BaseModel):
155186
llm_response: The required LLM response.
156187
sentiment: The optional sentiment.
157188
user_feedback: The optional user feedback.
189+
categories: The optional list of feedback categories (multi-select for negative feedback).
158190
159191
Example:
160192
```python
161193
feedback_request = FeedbackRequest(
162194
conversation_id="12345678-abcd-0000-0123-456789abcdef",
163195
user_question="what are you doing?",
164-
user_feedback="Great service!",
196+
user_feedback="This response is not helpful",
165197
llm_response="I don't know",
166198
sentiment=-1,
199+
categories=[FeedbackCategory.UNCLEAR, FeedbackCategory.TOO_BRIEF]
167200
)
168201
```
169202
"""
@@ -179,6 +212,12 @@ class FeedbackRequest(BaseModel):
179212
description="Feedback on the LLM response.",
180213
examples=["I'm not satisfied with the response because it is too vague."],
181214
)
215+
# Optional list of predefined feedback categories for negative feedback
216+
categories: Optional[list[FeedbackCategory]] = Field(
217+
default=None,
218+
description="List of feedback categories that describe issues with the LLM response (for negative feedback).",
219+
examples=[["unclear", "too_brief"]],
220+
)
182221

183222
# provides examples for /docs endpoint
184223
model_config = {
@@ -188,8 +227,23 @@ class FeedbackRequest(BaseModel):
188227
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
189228
"user_question": "foo",
190229
"llm_response": "bar",
191-
"user_feedback": "Great service!",
192-
"sentiment": 1,
230+
"user_feedback": "Not satisfied with the response quality.",
231+
"sentiment": -1,
232+
},
233+
{
234+
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
235+
"user_question": "What is the capital of France?",
236+
"llm_response": "The capital of France is Berlin.",
237+
"sentiment": -1,
238+
"categories": ["incorrect"],
239+
},
240+
{
241+
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
242+
"user_question": "How do I deploy a web app?",
243+
"llm_response": "Use Docker.",
244+
"user_feedback": "This response is too general and doesn't provide specific steps.",
245+
"sentiment": -1,
246+
"categories": ["too_brief", "incomplete", "missing_context"],
193247
}
194248
]
195249
}
@@ -213,9 +267,31 @@ def check_sentiment(cls, value: Optional[int]) -> Optional[int]:
213267
)
214268
return value
215269

270+
@field_validator("categories")
271+
@classmethod
272+
def validate_categories(cls, value: Optional[list[FeedbackCategory]]) -> Optional[list[FeedbackCategory]]:
273+
"""Validate feedback categories list."""
274+
if value is None:
275+
return value
276+
277+
if not isinstance(value, list):
278+
raise ValueError("Categories must be a list")
279+
280+
if len(value) == 0:
281+
return None # Convert empty list to None for consistency
282+
283+
unique_categories = list(set(value))
284+
return unique_categories
285+
216286
@model_validator(mode="after")
217-
def check_sentiment_or_user_feedback_set(self) -> Self:
218-
"""Ensure that either 'sentiment' or 'user_feedback' is set."""
219-
if self.sentiment is None and self.user_feedback is None:
220-
raise ValueError("Either 'sentiment' or 'user_feedback' must be set")
287+
def check_feedback_provided(self) -> Self:
288+
"""Ensure that at least one form of feedback is provided."""
289+
if (
290+
self.sentiment is None
291+
and self.user_feedback is None
292+
and self.categories is None
293+
):
294+
raise ValueError(
295+
"At least one form of feedback must be provided: 'sentiment', 'user_feedback', or 'categories'"
296+
)
221297
return self

0 commit comments

Comments
 (0)