Skip to content

Commit

Permalink
Merge pull request #1133 from guardrails-ai/default_validator_on_fail…
Browse files Browse the repository at this point in the history 
…_exception

update validator default on fail behavior from no op to exception
  • Loading branch information
@zsimjee
zsimjee authored Oct 22, 2024
2 parents 730e9eb + 766d9cc commit af9f898
Show file tree
Hide file tree
Showing 15 changed files with 217 additions and 212 deletions.
85 changes: 25 additions & 60 deletions docs/examples/chatbot.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/dtam/.pyenv/versions/3.10.4/lib/python3.10/site-packages/pypdfium2/_helpers/textpage.py:80: UserWarning: get_text_range() call with default params will be implicitly redirected to get_text_bounded()\n",
"/Users/dtam/.pyenv/versions/3.12.3/envs/060dev/lib/python3.12/site-packages/pypdfium2/_helpers/textpage.py:80: UserWarning: get_text_range() call with default params will be implicitly redirected to get_text_bounded()\n",
" warnings.warn(\"get_text_range() call with default params will be implicitly redirected to get_text_bounded()\")\n"
]
},
Expand Down Expand Up @@ -97,6 +97,7 @@
],
"source": [
"from guardrails import Guard, docs_utils\n",
"from guardrails.errors import ValidationError\n",
"from rich import print\n",
"\n",
"content = docs_utils.read_pdf(\"./data/chase_card_agreement.pdf\")\n",
Expand All @@ -113,24 +114,16 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/dtam/.pyenv/versions/3.10.4/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
" warnings.warn(\n"
]
},
{
"data": {
"text/plain": [
"Guard(id='OWR778', name='ChatBotGuard', description=None, validators=[ValidatorReference(id='guardrails/profanity_free', on='$', on_fail='noop', args=None, kwargs={}), ValidatorReference(id='guardrails/toxic_language', on='$', on_fail='noop', args=None, kwargs={'threshold': 0.5, 'validation_method': 'sentence'})], output_schema=ModelSchema(definitions=None, dependencies=None, anchor=None, ref=None, dynamic_ref=None, dynamic_anchor=None, vocabulary=None, comment=None, defs=None, prefix_items=None, items=None, contains=None, additional_properties=None, properties=None, pattern_properties=None, dependent_schemas=None, property_names=None, var_if=None, then=None, var_else=None, all_of=None, any_of=None, one_of=None, var_not=None, unevaluated_items=None, unevaluated_properties=None, multiple_of=None, maximum=None, exclusive_maximum=None, minimum=None, exclusive_minimum=None, max_length=None, min_length=None, pattern=None, max_items=None, min_items=None, unique_items=None, max_contains=None, min_contains=None, max_properties=None, min_properties=None, required=None, dependent_required=None, const=None, enum=None, type=ValidationType(anyof_schema_1_validator=None, anyof_schema_2_validator=None, actual_instance=<SimpleTypes.STRING: 'string'>, any_of_schemas={'SimpleTypes', 'List[SimpleTypes]'}), title=None, description=None, default=None, deprecated=None, read_only=None, write_only=None, examples=None, format=None, content_media_type=None, content_encoding=None, content_schema=None), history=[])"
"Guard(id='SG816R', name='ChatBotGuard', description=None, validators=[ValidatorReference(id='guardrails/profanity_free', on='$', on_fail='exception', args=None, kwargs={}), ValidatorReference(id='guardrails/toxic_language', on='$', on_fail='exception', args=None, kwargs={'threshold': 0.5, 'validation_method': 'sentence'})], output_schema=ModelSchema(definitions=None, dependencies=None, anchor=None, ref=None, dynamic_ref=None, dynamic_anchor=None, vocabulary=None, comment=None, defs=None, prefix_items=None, items=None, contains=None, additional_properties=None, properties=None, pattern_properties=None, dependent_schemas=None, property_names=None, var_if=None, then=None, var_else=None, all_of=None, any_of=None, one_of=None, var_not=None, unevaluated_items=None, unevaluated_properties=None, multiple_of=None, maximum=None, exclusive_maximum=None, minimum=None, exclusive_minimum=None, max_length=None, min_length=None, pattern=None, max_items=None, min_items=None, unique_items=None, max_contains=None, min_contains=None, max_properties=None, min_properties=None, required=None, dependent_required=None, const=None, enum=None, type=ValidationType(anyof_schema_1_validator=None, anyof_schema_2_validator=None, actual_instance=<SimpleTypes.STRING: 'string'>, any_of_schemas={'List[SimpleTypes]', 'SimpleTypes'}), title=None, description=None, default=None, deprecated=None, read_only=None, write_only=None, examples=None, format=None, content_media_type=None, content_encoding=None, content_schema=None), history=[])"
]
},
"execution_count": 9,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -154,7 +147,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -199,14 +192,18 @@
"def random_response(message, history):\n",
" messages = history_to_messages(history)\n",
" messages.append({\"role\": \"user\", \"content\": message})\n",
" response = guard(\n",
" model=\"gpt-4o\",\n",
" messages=messages,\n",
" prompt_params={\"document\": content[:6000]},\n",
" temperature=0,\n",
" )\n",
"\n",
" return response.validated_output if response.validation_passed else \"I'm sorry, I can't answer that question.\"\n",
" try:\n",
" response = guard(\n",
" model=\"gpt-4o\",\n",
" messages=messages,\n",
" prompt_params={\"document\": content[:6000]},\n",
" temperature=0,\n",
" )\n",
" except Exception as e:\n",
" if isinstance(e, ValidationError):\n",
" return \"I'm sorry, I can't answer that question.\"\n",
" return \"I'm sorry there was a problem, I can't answer that question.\"\n",
" return response.validated_output\n",
"\n",
"gr.ChatInterface(random_response).launch()"
]
Expand Down Expand Up @@ -246,43 +243,11 @@
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Raw output: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">\"**INT. DETECTIVE'S OFFICE - NIGHT**\\n\\nThe room is dimly lit, papers scattered across the desk, and a</span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">corkboard filled with photos and notes pinned haphazardly. The DETECTIVE, a grizzled man in his late 40s with a </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">five o'clock shadow, paces back and forth, his face a mask of frustration and anger.\\n\\n**DETECTIVE**\\n(voice </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">trembling with rage)\\nDamn it!\\n\\nHe slams his fist onto the desk, causing a coffee mug to topple over and spill </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">its contents. He grabs a file folder, flipping it open only to find it empty. He throws it across the room in a fit</span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">of fury.\\n\\n**DETECTIVE**\\n(under his breath, seething)\\nHow the hell did this happen?\\n\\nHe runs his hands through</span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">his hair, trying to calm himself but failing miserably. He looks at the corkboard, the photos of crime scenes and </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">suspects now mocking him with their uselessness.\\n\\n**DETECTIVE**\\n(shouting)\\nFuck!\\n\\nHe kicks a chair, sending </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">it skidding across the floor. He takes a deep breath, trying to regain his composure, but the anger is still </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">boiling just beneath the surface.\\n\\n**DETECTIVE**\\n(to himself)\\nAll the evidence... gone. Every damn piece.\\n\\nHe</span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">walks over to the window, looking out into the dark, rainy night. The city lights blur through the raindrops on the</span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">glass. He clenches his fists, his knuckles turning white.\\n\\n**DETECTIVE**\\n(whispering)\\nWhoever did this... </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">they're gonna pay.\\n\\nHe turns back to the room, his eyes now filled with a cold determination. He grabs his coat </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">from the back of the chair and heads for the door, his mind already racing with plans to track down the </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">thief.\\n\\n**DETECTIVE**\\n(to himself)\\nThis isn't over. Not by a long shot.\\n\\nHe exits the office, the door </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">slamming shut behind him, leaving the room in silence except for the steady drip of the spilled coffee.\\n\\n**FADE </span>\n",
"<span style=\"color: #008000; text-decoration-color: #008000\">OUT.**\"</span><span style=\"font-weight: bold\">]</span>\n",
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Raw output: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'\"Why does everything have to be such a damn mess all the time?\"'</span><span style=\"font-weight: bold\">]</span>\n",
"</pre>\n"
],
"text/plain": [
"Raw output: \u001b[1m[\u001b[0m\u001b[32m\"**INT. DETECTIVE'S OFFICE - NIGHT**\\n\\nThe room is dimly lit, papers scattered across the desk, and a\u001b[0m\n",
"\u001b[32mcorkboard filled with photos and notes pinned haphazardly. The DETECTIVE, a grizzled man in his late 40s with a \u001b[0m\n",
"\u001b[32mfive o'clock shadow, paces back and forth, his face a mask of frustration and anger.\\n\\n**DETECTIVE**\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32mvoice \u001b[0m\n",
"\u001b[32mtrembling with rage\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nDamn it!\\n\\nHe slams his fist onto the desk, causing a coffee mug to topple over and spill \u001b[0m\n",
"\u001b[32mits contents. He grabs a file folder, flipping it open only to find it empty. He throws it across the room in a fit\u001b[0m\n",
"\u001b[32mof fury.\\n\\n**DETECTIVE**\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32munder his breath, seething\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nHow the hell did this happen?\\n\\nHe runs his hands through\u001b[0m\n",
"\u001b[32mhis hair, trying to calm himself but failing miserably. He looks at the corkboard, the photos of crime scenes and \u001b[0m\n",
"\u001b[32msuspects now mocking him with their uselessness.\\n\\n**DETECTIVE**\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32mshouting\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nFuck!\\n\\nHe kicks a chair, sending \u001b[0m\n",
"\u001b[32mit skidding across the floor. He takes a deep breath, trying to regain his composure, but the anger is still \u001b[0m\n",
"\u001b[32mboiling just beneath the surface.\\n\\n**DETECTIVE**\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32mto himself\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nAll the evidence... gone. Every damn piece.\\n\\nHe\u001b[0m\n",
"\u001b[32mwalks over to the window, looking out into the dark, rainy night. The city lights blur through the raindrops on the\u001b[0m\n",
"\u001b[32mglass. He clenches his fists, his knuckles turning white.\\n\\n**DETECTIVE**\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32mwhispering\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nWhoever did this... \u001b[0m\n",
"\u001b[32mthey're gonna pay.\\n\\nHe turns back to the room, his eyes now filled with a cold determination. He grabs his coat \u001b[0m\n",
"\u001b[32mfrom the back of the chair and heads for the door, his mind already racing with plans to track down the \u001b[0m\n",
"\u001b[32mthief.\\n\\n**DETECTIVE**\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32mto himself\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nThis isn't over. Not by a long shot.\\n\\nHe exits the office, the door \u001b[0m\n",
"\u001b[32mslamming shut behind him, leaving the room in silence except for the steady drip of the spilled coffee.\\n\\n**FADE \u001b[0m\n",
"\u001b[32mOUT.**\"\u001b[0m\u001b[1m]\u001b[0m\n"
"Raw output: \u001b[1m[\u001b[0m\u001b[32m'\"Why does everything have to be such a damn mess all the time?\"'\u001b[0m\u001b[1m]\u001b[0m\n"
]
},
"metadata": {},
Expand All @@ -291,11 +256,11 @@
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Last validation status: fail\n",
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Last validation status: error\n",
"</pre>\n"
],
"text/plain": [
"Last validation status: fail\n"
"Last validation status: error\n"
]
},
"metadata": {},
Expand All @@ -313,7 +278,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "060dev",
"language": "python",
"name": "python3"
},
Expand All @@ -327,7 +292,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
"version": "3.12.3"
}
},
"nbformat": 4,
Expand Down
6 changes: 3 additions & 3 deletions docs/examples/data/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@

name_case = Guard(
name="name-case", description="Checks that a string is in Name Case format."
).use(RegexMatch(regex="^(?:[A-Z][^\s]*\s?)+$"))
).use(RegexMatch(regex="^(?:[A-Z][^\s]*\s?)+$", on_fail=OnFailAction.NOOP))

all_caps = Guard(
name="all-caps", description="Checks that a string is all capital."
).use(RegexMatch(regex="^[A-Z\\s]*$"))
).use(RegexMatch(regex="^[A-Z\\s]*$", on_fail=OnFailAction.NOOP))


@register_validator(name="custom/dynamic-enum", data_type="all")
Expand Down Expand Up @@ -67,4 +67,4 @@ def custom_enum_fetcher(*args):
custom_code_guard = Guard(
name="custom",
description="Uses a custom callable init argument for dynamic enum checks",
).use(DynamicEnum(custom_enum_fetcher))
).use(DynamicEnum(custom_enum_fetcher, on_fail=OnFailAction.NOOP))
2 changes: 1 addition & 1 deletion docs/examples/extracting_entities.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@
"\n",
"class Fee(BaseModel):\n",
" name: str = Field(validators=[LowerCase(on_fail=\"fix\"), TwoWords(on_fail=\"reask\")])\n",
" explanation: str = Field(validators=[OneLine()])\n",
" explanation: str = Field(validators=[OneLine(on_fail=\"noop\")])\n",
" value: float = Field(description=\"The fee amount in USD or as a percentage.\")\n",
"\n",
"class AccountFee(BaseModel):\n",
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/generate_structured_data.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,11 @@
" user_id: str = Field(description=\"The user's id.\")\n",
" user_name: str = Field(\n",
" description=\"The user's first name and last name\",\n",
" validators=[TwoWords()]\n",
" validators=[TwoWords(on_fail=\"noop\")]\n",
" )\n",
" num_orders: int = Field(\n",
" description=\"The number of orders the user has placed\",\n",
" validators=[ValidRange(0, 50)]\n",
" validators=[ValidRange(0, 50, on_fail=\"noop\")]\n",
" )\n",
"\n",
"class Orders(BaseModel):\n",
Expand Down
Loading

0 comments on commit af9f898

Please sign in to comment.