diff --git a/.stats.yml b/.stats.yml index ddfeae1..4082efa 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ configured_endpoints: 21 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/prompt-foundry%2Fprompt-foundry-sdk-441451c27073e45d1bdc832c5b66c26d90bd185bd94bd461b91257fbf0987ef2.yml +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/prompt-foundry%2Fprompt-foundry-sdk-2203adbcfccbd119a54a1fe9a29de02b1993184022804cfa4bc32947c8c1eb06.yml diff --git a/src/prompt_foundry_python_sdk/resources/evaluation_assertions.py b/src/prompt_foundry_python_sdk/resources/evaluation_assertions.py index 2c02422..c52d832 100644 --- a/src/prompt_foundry_python_sdk/resources/evaluation_assertions.py +++ b/src/prompt_foundry_python_sdk/resources/evaluation_assertions.py @@ -52,6 +52,7 @@ def create( type: Literal[ "CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH" ], + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -71,6 +72,8 @@ def create( type: The type of evaluation matcher to use. + weight: How heavily to weigh the assertion within the evaluation. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -88,6 +91,7 @@ def create( "target_value": target_value, "tool_name": tool_name, "type": type, + "weight": weight, }, evaluation_assertion_create_params.EvaluationAssertionCreateParams, ), @@ -108,6 +112,7 @@ def update( type: Literal[ "CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH" ], + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -127,6 +132,8 @@ def update( type: The type of evaluation matcher to use. + weight: How heavily to weigh the assertion within the evaluation. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -146,6 +153,7 @@ def update( "target_value": target_value, "tool_name": tool_name, "type": type, + "weight": weight, }, evaluation_assertion_update_params.EvaluationAssertionUpdateParams, ), @@ -280,6 +288,7 @@ async def create( type: Literal[ "CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH" ], + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -299,6 +308,8 @@ async def create( type: The type of evaluation matcher to use. + weight: How heavily to weigh the assertion within the evaluation. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -316,6 +327,7 @@ async def create( "target_value": target_value, "tool_name": tool_name, "type": type, + "weight": weight, }, evaluation_assertion_create_params.EvaluationAssertionCreateParams, ), @@ -336,6 +348,7 @@ async def update( type: Literal[ "CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH" ], + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -355,6 +368,8 @@ async def update( type: The type of evaluation matcher to use. + weight: How heavily to weigh the assertion within the evaluation. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -374,6 +389,7 @@ async def update( "target_value": target_value, "tool_name": tool_name, "type": type, + "weight": weight, }, evaluation_assertion_update_params.EvaluationAssertionUpdateParams, ), diff --git a/src/prompt_foundry_python_sdk/resources/evaluations.py b/src/prompt_foundry_python_sdk/resources/evaluations.py index bd3a1dd..8eaf74b 100644 --- a/src/prompt_foundry_python_sdk/resources/evaluations.py +++ b/src/prompt_foundry_python_sdk/resources/evaluations.py @@ -43,6 +43,8 @@ def create( appended_messages: Iterable[evaluation_create_params.AppendedMessage], prompt_id: str, variables: Dict[str, Optional[str]], + threshold: float | NotGiven = NOT_GIVEN, + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -58,6 +60,10 @@ def create( variables: The template variables added to the prompt when executing the prompt. + threshold: What percentage of assertions must pass for the evaluation to pass. + + weight: How heavily to weigh the evaluation within the prompt. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -73,6 +79,8 @@ def create( "appended_messages": appended_messages, "prompt_id": prompt_id, "variables": variables, + "threshold": threshold, + "weight": weight, }, evaluation_create_params.EvaluationCreateParams, ), @@ -89,6 +97,8 @@ def update( appended_messages: Iterable[evaluation_update_params.AppendedMessage], prompt_id: str, variables: Dict[str, Optional[str]], + threshold: float | NotGiven = NOT_GIVEN, + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -104,6 +114,10 @@ def update( variables: The template variables added to the prompt when executing the prompt. + threshold: What percentage of assertions must pass for the evaluation to pass. + + weight: How heavily to weigh the evaluation within the prompt. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -121,6 +135,8 @@ def update( "appended_messages": appended_messages, "prompt_id": prompt_id, "variables": variables, + "threshold": threshold, + "weight": weight, }, evaluation_update_params.EvaluationUpdateParams, ), @@ -231,6 +247,8 @@ async def create( appended_messages: Iterable[evaluation_create_params.AppendedMessage], prompt_id: str, variables: Dict[str, Optional[str]], + threshold: float | NotGiven = NOT_GIVEN, + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -246,6 +264,10 @@ async def create( variables: The template variables added to the prompt when executing the prompt. + threshold: What percentage of assertions must pass for the evaluation to pass. + + weight: How heavily to weigh the evaluation within the prompt. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -261,6 +283,8 @@ async def create( "appended_messages": appended_messages, "prompt_id": prompt_id, "variables": variables, + "threshold": threshold, + "weight": weight, }, evaluation_create_params.EvaluationCreateParams, ), @@ -277,6 +301,8 @@ async def update( appended_messages: Iterable[evaluation_update_params.AppendedMessage], prompt_id: str, variables: Dict[str, Optional[str]], + threshold: float | NotGiven = NOT_GIVEN, + weight: float | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -292,6 +318,10 @@ async def update( variables: The template variables added to the prompt when executing the prompt. + threshold: What percentage of assertions must pass for the evaluation to pass. + + weight: How heavily to weigh the evaluation within the prompt. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -309,6 +339,8 @@ async def update( "appended_messages": appended_messages, "prompt_id": prompt_id, "variables": variables, + "threshold": threshold, + "weight": weight, }, evaluation_update_params.EvaluationUpdateParams, ), diff --git a/src/prompt_foundry_python_sdk/types/evaluation.py b/src/prompt_foundry_python_sdk/types/evaluation.py index 5d1e372..8873443 100644 --- a/src/prompt_foundry_python_sdk/types/evaluation.py +++ b/src/prompt_foundry_python_sdk/types/evaluation.py @@ -99,3 +99,9 @@ class Evaluation(BaseModel): variables: Dict[str, Optional[str]] """The template variables added to the prompt when executing the prompt.""" + + threshold: Optional[float] = None + """What percentage of assertions must pass for the evaluation to pass.""" + + weight: Optional[float] = None + """How heavily to weigh the evaluation within the prompt.""" diff --git a/src/prompt_foundry_python_sdk/types/evaluation_assertion.py b/src/prompt_foundry_python_sdk/types/evaluation_assertion.py index f6d44df..5eeaea0 100644 --- a/src/prompt_foundry_python_sdk/types/evaluation_assertion.py +++ b/src/prompt_foundry_python_sdk/types/evaluation_assertion.py @@ -31,3 +31,6 @@ class EvaluationAssertion(BaseModel): type: Literal["CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH"] """The type of evaluation matcher to use.""" + + weight: Optional[float] = None + """How heavily to weigh the assertion within the evaluation.""" diff --git a/src/prompt_foundry_python_sdk/types/evaluation_assertion_create_params.py b/src/prompt_foundry_python_sdk/types/evaluation_assertion_create_params.py index 3be61b8..3adaf1e 100644 --- a/src/prompt_foundry_python_sdk/types/evaluation_assertion_create_params.py +++ b/src/prompt_foundry_python_sdk/types/evaluation_assertion_create_params.py @@ -31,3 +31,6 @@ class EvaluationAssertionCreateParams(TypedDict, total=False): Literal["CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH"] ] """The type of evaluation matcher to use.""" + + weight: float + """How heavily to weigh the assertion within the evaluation.""" diff --git a/src/prompt_foundry_python_sdk/types/evaluation_assertion_update_params.py b/src/prompt_foundry_python_sdk/types/evaluation_assertion_update_params.py index 555bfaf..8c9813c 100644 --- a/src/prompt_foundry_python_sdk/types/evaluation_assertion_update_params.py +++ b/src/prompt_foundry_python_sdk/types/evaluation_assertion_update_params.py @@ -31,3 +31,6 @@ class EvaluationAssertionUpdateParams(TypedDict, total=False): Literal["CONTAINS", "EXACT_MATCH", "JSON_CONTAINS", "JSON_EXACT_MATCH", "TOOL_CALLED", "TOOL_CALLED_WITH"] ] """The type of evaluation matcher to use.""" + + weight: float + """How heavily to weigh the assertion within the evaluation.""" diff --git a/src/prompt_foundry_python_sdk/types/evaluation_create_params.py b/src/prompt_foundry_python_sdk/types/evaluation_create_params.py index dcd279f..7105787 100644 --- a/src/prompt_foundry_python_sdk/types/evaluation_create_params.py +++ b/src/prompt_foundry_python_sdk/types/evaluation_create_params.py @@ -29,6 +29,12 @@ class EvaluationCreateParams(TypedDict, total=False): variables: Required[Dict[str, Optional[str]]] """The template variables added to the prompt when executing the prompt.""" + threshold: float + """What percentage of assertions must pass for the evaluation to pass.""" + + weight: float + """How heavily to weigh the evaluation within the prompt.""" + class AppendedMessageContentTextContentBlockSchema(TypedDict, total=False): text: Required[str] diff --git a/src/prompt_foundry_python_sdk/types/evaluation_update_params.py b/src/prompt_foundry_python_sdk/types/evaluation_update_params.py index 13b8939..8097a05 100644 --- a/src/prompt_foundry_python_sdk/types/evaluation_update_params.py +++ b/src/prompt_foundry_python_sdk/types/evaluation_update_params.py @@ -29,6 +29,12 @@ class EvaluationUpdateParams(TypedDict, total=False): variables: Required[Dict[str, Optional[str]]] """The template variables added to the prompt when executing the prompt.""" + threshold: float + """What percentage of assertions must pass for the evaluation to pass.""" + + weight: float + """How heavily to weigh the evaluation within the prompt.""" + class AppendedMessageContentTextContentBlockSchema(TypedDict, total=False): text: Required[str] diff --git a/tests/api_resources/test_evaluation_assertions.py b/tests/api_resources/test_evaluation_assertions.py index 58f1aea..302c562 100644 --- a/tests/api_resources/test_evaluation_assertions.py +++ b/tests/api_resources/test_evaluation_assertions.py @@ -32,6 +32,18 @@ def test_method_create(self, client: PromptFoundry) -> None: ) assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize + def test_method_create_with_all_params(self, client: PromptFoundry) -> None: + evaluation_assertion = client.evaluation_assertions.create( + evaluation_id="evaluationId", + json_path="jsonPath", + target_value="targetValue", + tool_name="toolName", + type="CONTAINS", + weight=0, + ) + assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize def test_raw_response_create(self, client: PromptFoundry) -> None: response = client.evaluation_assertions.with_raw_response.create( @@ -76,6 +88,19 @@ def test_method_update(self, client: PromptFoundry) -> None: ) assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize + def test_method_update_with_all_params(self, client: PromptFoundry) -> None: + evaluation_assertion = client.evaluation_assertions.update( + id="1212121", + evaluation_id="evaluationId", + json_path="jsonPath", + target_value="targetValue", + tool_name="toolName", + type="CONTAINS", + weight=0, + ) + assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize def test_raw_response_update(self, client: PromptFoundry) -> None: response = client.evaluation_assertions.with_raw_response.update( @@ -245,6 +270,18 @@ async def test_method_create(self, async_client: AsyncPromptFoundry) -> None: ) assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize + async def test_method_create_with_all_params(self, async_client: AsyncPromptFoundry) -> None: + evaluation_assertion = await async_client.evaluation_assertions.create( + evaluation_id="evaluationId", + json_path="jsonPath", + target_value="targetValue", + tool_name="toolName", + type="CONTAINS", + weight=0, + ) + assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize async def test_raw_response_create(self, async_client: AsyncPromptFoundry) -> None: response = await async_client.evaluation_assertions.with_raw_response.create( @@ -289,6 +326,19 @@ async def test_method_update(self, async_client: AsyncPromptFoundry) -> None: ) assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize + async def test_method_update_with_all_params(self, async_client: AsyncPromptFoundry) -> None: + evaluation_assertion = await async_client.evaluation_assertions.update( + id="1212121", + evaluation_id="evaluationId", + json_path="jsonPath", + target_value="targetValue", + tool_name="toolName", + type="CONTAINS", + weight=0, + ) + assert_matches_type(EvaluationAssertion, evaluation_assertion, path=["response"]) + @parametrize async def test_raw_response_update(self, async_client: AsyncPromptFoundry) -> None: response = await async_client.evaluation_assertions.with_raw_response.update( diff --git a/tests/api_resources/test_evaluations.py b/tests/api_resources/test_evaluations.py index c6f8555..a226be1 100644 --- a/tests/api_resources/test_evaluations.py +++ b/tests/api_resources/test_evaluations.py @@ -82,6 +82,69 @@ def test_method_create(self, client: PromptFoundry) -> None: ) assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize + def test_method_create_with_all_params(self, client: PromptFoundry) -> None: + evaluation = client.evaluations.create( + appended_messages=[ + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + ], + prompt_id="promptId", + variables={"foo": "string"}, + threshold=0, + weight=0, + ) + assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize def test_raw_response_create(self, client: PromptFoundry) -> None: response = client.evaluations.with_raw_response.create( @@ -276,6 +339,70 @@ def test_method_update(self, client: PromptFoundry) -> None: ) assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize + def test_method_update_with_all_params(self, client: PromptFoundry) -> None: + evaluation = client.evaluations.update( + id="1212121", + appended_messages=[ + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + ], + prompt_id="promptId", + variables={"foo": "string"}, + threshold=0, + weight=0, + ) + assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize def test_raw_response_update(self, client: PromptFoundry) -> None: response = client.evaluations.with_raw_response.update( @@ -638,6 +765,69 @@ async def test_method_create(self, async_client: AsyncPromptFoundry) -> None: ) assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize + async def test_method_create_with_all_params(self, async_client: AsyncPromptFoundry) -> None: + evaluation = await async_client.evaluations.create( + appended_messages=[ + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + ], + prompt_id="promptId", + variables={"foo": "string"}, + threshold=0, + weight=0, + ) + assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize async def test_raw_response_create(self, async_client: AsyncPromptFoundry) -> None: response = await async_client.evaluations.with_raw_response.create( @@ -832,6 +1022,70 @@ async def test_method_update(self, async_client: AsyncPromptFoundry) -> None: ) assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize + async def test_method_update_with_all_params(self, async_client: AsyncPromptFoundry) -> None: + evaluation = await async_client.evaluations.update( + id="1212121", + appended_messages=[ + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + { + "type": "TEXT", + "text": "text", + }, + ], + "role": "assistant", + }, + ], + prompt_id="promptId", + variables={"foo": "string"}, + threshold=0, + weight=0, + ) + assert_matches_type(Evaluation, evaluation, path=["response"]) + @parametrize async def test_raw_response_update(self, async_client: AsyncPromptFoundry) -> None: response = await async_client.evaluations.with_raw_response.update(