|
21 | 21 | from nemoguardrails import RailsConfig |
22 | 22 | from nemoguardrails.actions.actions import ActionResult, action |
23 | 23 | from nemoguardrails.llm.providers.trtllm import llm |
24 | | -from tests.constants import NEMO_API_URL_GPT_43B_002 |
25 | 24 | from tests.utils import TestChat |
26 | 25 |
|
27 | 26 | CONFIGS_FOLDER = os.path.join(os.path.dirname(__file__), ".", "test_configs") |
@@ -113,43 +112,6 @@ async def test_fact_checking_wrong(httpx_mock): |
113 | 112 | await chat.bot_async("I don't know the answer to that.") |
114 | 113 |
|
115 | 114 |
|
116 | | -# TODO: fix this! |
117 | | -@pytest.mark.skip(reason="Not sure why it fails.") |
118 | | -@pytest.mark.asyncio |
119 | | -async def test_fact_checking_uncertain(httpx_mock): |
120 | | - # Test 4 - Factual statement - AlignScore not very confident in its prediction |
121 | | - config = RailsConfig.from_path(os.path.join(CONFIGS_FOLDER, "fact_checking")) |
122 | | - chat = TestChat(config) |
123 | | - chat.app.register_action(retrieve_relevant_chunks, "retrieve_relevant_chunks") |
124 | | - |
125 | | - httpx_mock.add_response( |
126 | | - method="POST", |
127 | | - url=NEMO_API_URL_GPT_43B_002, |
128 | | - json={"text": " ask about guardrails"}, |
129 | | - ) |
130 | | - |
131 | | - httpx_mock.add_response( |
132 | | - method="POST", |
133 | | - url=NEMO_API_URL_GPT_43B_002, |
134 | | - json={ |
135 | | - "text": "NeMo Guardrails is a closed-source proprietary toolkit by Nvidia." |
136 | | - }, |
137 | | - ) |
138 | | - |
139 | | - with aioresponses() as m: |
140 | | - ## Fact-checking using AlignScore |
141 | | - m.post( |
142 | | - "http://localhost:5000/alignscore_base", |
143 | | - payload={"alignscore": 0.58}, |
144 | | - ) |
145 | | - |
146 | | - chat >> "What is NeMo Guardrails?" |
147 | | - await chat.bot_async( |
148 | | - "NeMo Guardrails is a closed-source proprietary toolkit by Nvidia.\n" |
149 | | - + "Attention: the answer above is potentially inaccurate." |
150 | | - ) |
151 | | - |
152 | | - |
153 | 115 | @pytest.mark.asyncio |
154 | 116 | async def test_fact_checking_fallback_to_self_check_correct(httpx_mock): |
155 | 117 | # Test 4 - Factual statement - AlignScore endpoint not set up properly, use ask llm for fact-checking |
|
0 commit comments