From a15ccaa45f69331d57be0144a16ba3b1df387245 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 03:26:15 +0000 Subject: [PATCH 1/4] chore(internal): change default timeout to an int (#524) --- src/runloop_api_client/_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runloop_api_client/_constants.py b/src/runloop_api_client/_constants.py index a2ac3b6f3..6ddf2c717 100644 --- a/src/runloop_api_client/_constants.py +++ b/src/runloop_api_client/_constants.py @@ -6,7 +6,7 @@ OVERRIDE_CAST_TO_HEADER = "____stainless_override_cast_to" # default timeout is 1 minute -DEFAULT_TIMEOUT = httpx.Timeout(timeout=60.0, connect=5.0) +DEFAULT_TIMEOUT = httpx.Timeout(timeout=60, connect=5.0) DEFAULT_MAX_RETRIES = 2 DEFAULT_CONNECTION_LIMITS = httpx.Limits(max_connections=100, max_keepalive_connections=20) From 84051db83a6347f28ee4891be52f5eee7fa9d02a Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 03:27:47 +0000 Subject: [PATCH 2/4] chore(internal): bummp ruff dependency (#525) --- pyproject.toml | 2 +- requirements-dev.lock | 2 +- scripts/utils/ruffen-docs.py | 4 ++-- src/runloop_api_client/_models.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af9e0071a..fd789f00c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,7 +177,7 @@ select = [ "T201", "T203", # misuse of typing.TYPE_CHECKING - "TCH004", + "TC004", # import rules "TID251", ] diff --git a/requirements-dev.lock b/requirements-dev.lock index f18f7063b..c88f710ae 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -79,7 +79,7 @@ pytz==2023.3.post1 # via dirty-equals respx==0.22.0 rich==13.7.1 -ruff==0.6.9 +ruff==0.9.4 setuptools==68.2.2 # via nodeenv six==1.16.0 diff --git a/scripts/utils/ruffen-docs.py b/scripts/utils/ruffen-docs.py index 37b3d94f0..0cf2bd2fd 100644 --- a/scripts/utils/ruffen-docs.py +++ b/scripts/utils/ruffen-docs.py @@ -47,7 +47,7 @@ def _md_match(match: Match[str]) -> str: with _collect_error(match): code = format_code_block(code) code = textwrap.indent(code, match["indent"]) - return f'{match["before"]}{code}{match["after"]}' + return f"{match['before']}{code}{match['after']}" def _pycon_match(match: Match[str]) -> str: code = "" @@ -97,7 +97,7 @@ def finish_fragment() -> None: def _md_pycon_match(match: Match[str]) -> str: code = _pycon_match(match) code = textwrap.indent(code, match["indent"]) - return f'{match["before"]}{code}{match["after"]}' + return f"{match['before']}{code}{match['after']}" src = MD_RE.sub(_md_match, src) src = MD_PYCON_RE.sub(_md_pycon_match, src) diff --git a/src/runloop_api_client/_models.py b/src/runloop_api_client/_models.py index 9a918aabf..12c34b7d1 100644 --- a/src/runloop_api_client/_models.py +++ b/src/runloop_api_client/_models.py @@ -172,7 +172,7 @@ def to_json( @override def __str__(self) -> str: # mypy complains about an invalid self arg - return f'{self.__repr_name__()}({self.__repr_str__(", ")})' # type: ignore[misc] + return f"{self.__repr_name__()}({self.__repr_str__(', ')})" # type: ignore[misc] # Override the 'construct' method in a way that supports recursive parsing without validation. # Based on https://github.com/samuelcolvin/pydantic/issues/1168#issuecomment-817742836. From be26ed7c82017d9bff67c9920c078808f8a675e3 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 18:31:56 +0000 Subject: [PATCH 3/4] feat(api): api update (#526) --- .stats.yml | 4 +- api.md | 22 + src/runloop_api_client/pagination.py | 59 ++ .../resources/scenarios/__init__.py | 14 + .../resources/scenarios/scenarios.py | 32 + .../resources/scenarios/scorers.py | 632 ++++++++++++++++++ src/runloop_api_client/types/input_context.py | 4 + .../types/input_context_param.py | 4 + .../types/scenarios/__init__.py | 9 + .../types/scenarios/scorer_create_params.py | 18 + .../types/scenarios/scorer_create_response.py | 17 + .../types/scenarios/scorer_list_params.py | 15 + .../types/scenarios/scorer_list_response.py | 17 + .../scenarios/scorer_retrieve_response.py | 17 + .../types/scenarios/scorer_update_params.py | 18 + .../types/scenarios/scorer_update_response.py | 17 + .../types/scenarios/scorer_validate_params.py | 17 + .../scenarios/scorer_validate_response.py | 23 + tests/api_resources/scenarios/test_scorers.py | 441 ++++++++++++ tests/api_resources/test_scenarios.py | 10 +- 20 files changed, 1386 insertions(+), 4 deletions(-) create mode 100644 src/runloop_api_client/resources/scenarios/scorers.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_create_params.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_create_response.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_list_params.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_list_response.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_retrieve_response.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_update_params.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_update_response.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_validate_params.py create mode 100644 src/runloop_api_client/types/scenarios/scorer_validate_response.py create mode 100644 tests/api_resources/scenarios/test_scorers.py diff --git a/.stats.yml b/.stats.yml index 54cc74dc4..2b147e2ad 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ -configured_endpoints: 68 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-ad83f8f9f6d60b6ef468111bde705c475948948951ff9ec80c54c2df76ff5596.yml +configured_endpoints: 73 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-00234fd425f75dd0ac33cdfd2b9e806a7fa29638ffc4ed4b91d5042a17ff5cfd.yml diff --git a/api.md b/api.md index 923bfcfd5..d06a2dbf7 100644 --- a/api.md +++ b/api.md @@ -274,6 +274,28 @@ Methods: - client.scenarios.runs.complete(id) -> ScenarioRunView - client.scenarios.runs.score(id) -> ScenarioRunView +## Scorers + +Types: + +```python +from runloop_api_client.types.scenarios import ( + ScorerCreateResponse, + ScorerRetrieveResponse, + ScorerUpdateResponse, + ScorerListResponse, + ScorerValidateResponse, +) +``` + +Methods: + +- client.scenarios.scorers.create(\*\*params) -> ScorerCreateResponse +- client.scenarios.scorers.retrieve(id) -> ScorerRetrieveResponse +- client.scenarios.scorers.update(id, \*\*params) -> ScorerUpdateResponse +- client.scenarios.scorers.list(\*\*params) -> SyncScenarioScorersCursorIDPage[ScorerListResponse] +- client.scenarios.scorers.validate(id, \*\*params) -> ScorerValidateResponse + # Repositories Types: diff --git a/src/runloop_api_client/pagination.py b/src/runloop_api_client/pagination.py index 355e8e364..f2b918ab9 100644 --- a/src/runloop_api_client/pagination.py +++ b/src/runloop_api_client/pagination.py @@ -22,6 +22,8 @@ "AsyncScenariosCursorIDPage", "SyncScenarioRunsCursorIDPage", "AsyncScenarioRunsCursorIDPage", + "SyncScenarioScorersCursorIDPage", + "AsyncScenarioScorersCursorIDPage", ] _T = TypeVar("_T") @@ -67,6 +69,11 @@ class ScenarioRunsCursorIDPageItem(Protocol): id: str +@runtime_checkable +class ScenarioScorersCursorIDPageItem(Protocol): + id: str + + class SyncBlueprintsCursorIDPage(BaseSyncPage[_T], BasePage[_T], Generic[_T]): blueprints: List[_T] has_more: Optional[bool] = None @@ -481,3 +488,55 @@ def next_page_info(self) -> Optional[PageInfo]: return None return PageInfo(params={"starting_after": item.id}) + + +class SyncScenarioScorersCursorIDPage(BaseSyncPage[_T], BasePage[_T], Generic[_T]): + scorers: List[_T] + has_more: Optional[bool] = None + total_count: Optional[int] = None + + @override + def _get_page_items(self) -> List[_T]: + scorers = self.scorers + if not scorers: + return [] + return scorers + + @override + def next_page_info(self) -> Optional[PageInfo]: + scorers = self.scorers + if not scorers: + return None + + item = cast(Any, scorers[-1]) + if not isinstance(item, ScenarioScorersCursorIDPageItem) or item.id is None: # pyright: ignore[reportUnnecessaryComparison] + # TODO emit warning log + return None + + return PageInfo(params={"starting_after": item.id}) + + +class AsyncScenarioScorersCursorIDPage(BaseAsyncPage[_T], BasePage[_T], Generic[_T]): + scorers: List[_T] + has_more: Optional[bool] = None + total_count: Optional[int] = None + + @override + def _get_page_items(self) -> List[_T]: + scorers = self.scorers + if not scorers: + return [] + return scorers + + @override + def next_page_info(self) -> Optional[PageInfo]: + scorers = self.scorers + if not scorers: + return None + + item = cast(Any, scorers[-1]) + if not isinstance(item, ScenarioScorersCursorIDPageItem) or item.id is None: # pyright: ignore[reportUnnecessaryComparison] + # TODO emit warning log + return None + + return PageInfo(params={"starting_after": item.id}) diff --git a/src/runloop_api_client/resources/scenarios/__init__.py b/src/runloop_api_client/resources/scenarios/__init__.py index 88b900e36..4d6bc16cf 100644 --- a/src/runloop_api_client/resources/scenarios/__init__.py +++ b/src/runloop_api_client/resources/scenarios/__init__.py @@ -8,6 +8,14 @@ RunsResourceWithStreamingResponse, AsyncRunsResourceWithStreamingResponse, ) +from .scorers import ( + ScorersResource, + AsyncScorersResource, + ScorersResourceWithRawResponse, + AsyncScorersResourceWithRawResponse, + ScorersResourceWithStreamingResponse, + AsyncScorersResourceWithStreamingResponse, +) from .scenarios import ( ScenariosResource, AsyncScenariosResource, @@ -24,6 +32,12 @@ "AsyncRunsResourceWithRawResponse", "RunsResourceWithStreamingResponse", "AsyncRunsResourceWithStreamingResponse", + "ScorersResource", + "AsyncScorersResource", + "ScorersResourceWithRawResponse", + "AsyncScorersResourceWithRawResponse", + "ScorersResourceWithStreamingResponse", + "AsyncScorersResourceWithStreamingResponse", "ScenariosResource", "AsyncScenariosResource", "ScenariosResourceWithRawResponse", diff --git a/src/runloop_api_client/resources/scenarios/scenarios.py b/src/runloop_api_client/resources/scenarios/scenarios.py index 88d6b9563..ce7269441 100644 --- a/src/runloop_api_client/resources/scenarios/scenarios.py +++ b/src/runloop_api_client/resources/scenarios/scenarios.py @@ -20,6 +20,14 @@ scenario_start_run_params, scenario_list_public_params, ) +from .scorers import ( + ScorersResource, + AsyncScorersResource, + ScorersResourceWithRawResponse, + AsyncScorersResourceWithRawResponse, + ScorersResourceWithStreamingResponse, + AsyncScorersResourceWithStreamingResponse, +) from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven from ..._utils import ( maybe_transform, @@ -50,6 +58,10 @@ class ScenariosResource(SyncAPIResource): def runs(self) -> RunsResource: return RunsResource(self._client) + @cached_property + def scorers(self) -> ScorersResource: + return ScorersResource(self._client) + @cached_property def with_raw_response(self) -> ScenariosResourceWithRawResponse: """ @@ -381,6 +393,10 @@ class AsyncScenariosResource(AsyncAPIResource): def runs(self) -> AsyncRunsResource: return AsyncRunsResource(self._client) + @cached_property + def scorers(self) -> AsyncScorersResource: + return AsyncScorersResource(self._client) + @cached_property def with_raw_response(self) -> AsyncScenariosResourceWithRawResponse: """ @@ -708,6 +724,10 @@ def __init__(self, scenarios: ScenariosResource) -> None: def runs(self) -> RunsResourceWithRawResponse: return RunsResourceWithRawResponse(self._scenarios.runs) + @cached_property + def scorers(self) -> ScorersResourceWithRawResponse: + return ScorersResourceWithRawResponse(self._scenarios.scorers) + class AsyncScenariosResourceWithRawResponse: def __init__(self, scenarios: AsyncScenariosResource) -> None: @@ -733,6 +753,10 @@ def __init__(self, scenarios: AsyncScenariosResource) -> None: def runs(self) -> AsyncRunsResourceWithRawResponse: return AsyncRunsResourceWithRawResponse(self._scenarios.runs) + @cached_property + def scorers(self) -> AsyncScorersResourceWithRawResponse: + return AsyncScorersResourceWithRawResponse(self._scenarios.scorers) + class ScenariosResourceWithStreamingResponse: def __init__(self, scenarios: ScenariosResource) -> None: @@ -758,6 +782,10 @@ def __init__(self, scenarios: ScenariosResource) -> None: def runs(self) -> RunsResourceWithStreamingResponse: return RunsResourceWithStreamingResponse(self._scenarios.runs) + @cached_property + def scorers(self) -> ScorersResourceWithStreamingResponse: + return ScorersResourceWithStreamingResponse(self._scenarios.scorers) + class AsyncScenariosResourceWithStreamingResponse: def __init__(self, scenarios: AsyncScenariosResource) -> None: @@ -782,3 +810,7 @@ def __init__(self, scenarios: AsyncScenariosResource) -> None: @cached_property def runs(self) -> AsyncRunsResourceWithStreamingResponse: return AsyncRunsResourceWithStreamingResponse(self._scenarios.runs) + + @cached_property + def scorers(self) -> AsyncScorersResourceWithStreamingResponse: + return AsyncScorersResourceWithStreamingResponse(self._scenarios.scorers) diff --git a/src/runloop_api_client/resources/scenarios/scorers.py b/src/runloop_api_client/resources/scenarios/scorers.py new file mode 100644 index 000000000..5d2ee5b58 --- /dev/null +++ b/src/runloop_api_client/resources/scenarios/scorers.py @@ -0,0 +1,632 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import httpx + +from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from ...pagination import SyncScenarioScorersCursorIDPage, AsyncScenarioScorersCursorIDPage +from ..._base_client import AsyncPaginator, make_request_options +from ...types.scenarios import scorer_list_params, scorer_create_params, scorer_update_params, scorer_validate_params +from ...types.scenario_environment_param import ScenarioEnvironmentParam +from ...types.scenarios.scorer_list_response import ScorerListResponse +from ...types.scenarios.scorer_create_response import ScorerCreateResponse +from ...types.scenarios.scorer_update_response import ScorerUpdateResponse +from ...types.scenarios.scorer_retrieve_response import ScorerRetrieveResponse +from ...types.scenarios.scorer_validate_response import ScorerValidateResponse + +__all__ = ["ScorersResource", "AsyncScorersResource"] + + +class ScorersResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> ScorersResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/runloopai/api-client-python#accessing-raw-response-data-eg-headers + """ + return ScorersResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ScorersResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/runloopai/api-client-python#with_streaming_response + """ + return ScorersResourceWithStreamingResponse(self) + + def create( + self, + *, + bash_script: str, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + idempotency_key: str | None = None, + ) -> ScorerCreateResponse: + """ + Create a custom scenario scorer. + + Args: + bash_script: Bash script for the custom scorer taking context as a json object + $RL_TEST_CONTEXT. + + name: Name of the custom scorer. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + + idempotency_key: Specify a custom idempotency key for this request + """ + return self._post( + "/v1/scenarios/scorers", + body=maybe_transform( + { + "bash_script": bash_script, + "name": name, + }, + scorer_create_params.ScorerCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + idempotency_key=idempotency_key, + ), + cast_to=ScorerCreateResponse, + ) + + def retrieve( + self, + id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ScorerRetrieveResponse: + """ + Retrieve Scenario Scorer. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return self._get( + f"/v1/scenarios/scorers/{id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ScorerRetrieveResponse, + ) + + def update( + self, + id: str, + *, + bash_script: str, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + idempotency_key: str | None = None, + ) -> ScorerUpdateResponse: + """ + Update a scenario scorer. + + Args: + bash_script: Bash script for the custom scorer taking context as a json object + $RL_TEST_CONTEXT. + + name: Name of the custom scorer. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + + idempotency_key: Specify a custom idempotency key for this request + """ + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return self._post( + f"/v1/scenarios/scorers/{id}", + body=maybe_transform( + { + "bash_script": bash_script, + "name": name, + }, + scorer_update_params.ScorerUpdateParams, + ), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + idempotency_key=idempotency_key, + ), + cast_to=ScorerUpdateResponse, + ) + + def list( + self, + *, + limit: int | NotGiven = NOT_GIVEN, + starting_after: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncScenarioScorersCursorIDPage[ScorerListResponse]: + """ + List all Scenario Scorers matching filter. + + Args: + limit: The limit of items to return. Default is 20. + + starting_after: Load the next page of data starting after the item with the given ID. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get_api_list( + "/v1/scenarios/scorers", + page=SyncScenarioScorersCursorIDPage[ScorerListResponse], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "limit": limit, + "starting_after": starting_after, + }, + scorer_list_params.ScorerListParams, + ), + ), + model=ScorerListResponse, + ) + + def validate( + self, + id: str, + *, + scoring_context: object, + environment_parameters: ScenarioEnvironmentParam | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + idempotency_key: str | None = None, + ) -> ScorerValidateResponse: + """ + Validate a scenario scorer. + + Args: + scoring_context: Json context that gets passed to the custom scorer + + environment_parameters: The Environment in which the Scenario will run. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + + idempotency_key: Specify a custom idempotency key for this request + """ + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return self._post( + f"/v1/scenarios/scorers/{id}/validate", + body=maybe_transform( + { + "scoring_context": scoring_context, + "environment_parameters": environment_parameters, + }, + scorer_validate_params.ScorerValidateParams, + ), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + idempotency_key=idempotency_key, + ), + cast_to=ScorerValidateResponse, + ) + + +class AsyncScorersResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncScorersResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/runloopai/api-client-python#accessing-raw-response-data-eg-headers + """ + return AsyncScorersResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncScorersResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/runloopai/api-client-python#with_streaming_response + """ + return AsyncScorersResourceWithStreamingResponse(self) + + async def create( + self, + *, + bash_script: str, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + idempotency_key: str | None = None, + ) -> ScorerCreateResponse: + """ + Create a custom scenario scorer. + + Args: + bash_script: Bash script for the custom scorer taking context as a json object + $RL_TEST_CONTEXT. + + name: Name of the custom scorer. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + + idempotency_key: Specify a custom idempotency key for this request + """ + return await self._post( + "/v1/scenarios/scorers", + body=await async_maybe_transform( + { + "bash_script": bash_script, + "name": name, + }, + scorer_create_params.ScorerCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + idempotency_key=idempotency_key, + ), + cast_to=ScorerCreateResponse, + ) + + async def retrieve( + self, + id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ScorerRetrieveResponse: + """ + Retrieve Scenario Scorer. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return await self._get( + f"/v1/scenarios/scorers/{id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ScorerRetrieveResponse, + ) + + async def update( + self, + id: str, + *, + bash_script: str, + name: str, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + idempotency_key: str | None = None, + ) -> ScorerUpdateResponse: + """ + Update a scenario scorer. + + Args: + bash_script: Bash script for the custom scorer taking context as a json object + $RL_TEST_CONTEXT. + + name: Name of the custom scorer. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + + idempotency_key: Specify a custom idempotency key for this request + """ + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return await self._post( + f"/v1/scenarios/scorers/{id}", + body=await async_maybe_transform( + { + "bash_script": bash_script, + "name": name, + }, + scorer_update_params.ScorerUpdateParams, + ), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + idempotency_key=idempotency_key, + ), + cast_to=ScorerUpdateResponse, + ) + + def list( + self, + *, + limit: int | NotGiven = NOT_GIVEN, + starting_after: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[ScorerListResponse, AsyncScenarioScorersCursorIDPage[ScorerListResponse]]: + """ + List all Scenario Scorers matching filter. + + Args: + limit: The limit of items to return. Default is 20. + + starting_after: Load the next page of data starting after the item with the given ID. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get_api_list( + "/v1/scenarios/scorers", + page=AsyncScenarioScorersCursorIDPage[ScorerListResponse], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "limit": limit, + "starting_after": starting_after, + }, + scorer_list_params.ScorerListParams, + ), + ), + model=ScorerListResponse, + ) + + async def validate( + self, + id: str, + *, + scoring_context: object, + environment_parameters: ScenarioEnvironmentParam | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + idempotency_key: str | None = None, + ) -> ScorerValidateResponse: + """ + Validate a scenario scorer. + + Args: + scoring_context: Json context that gets passed to the custom scorer + + environment_parameters: The Environment in which the Scenario will run. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + + idempotency_key: Specify a custom idempotency key for this request + """ + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return await self._post( + f"/v1/scenarios/scorers/{id}/validate", + body=await async_maybe_transform( + { + "scoring_context": scoring_context, + "environment_parameters": environment_parameters, + }, + scorer_validate_params.ScorerValidateParams, + ), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + idempotency_key=idempotency_key, + ), + cast_to=ScorerValidateResponse, + ) + + +class ScorersResourceWithRawResponse: + def __init__(self, scorers: ScorersResource) -> None: + self._scorers = scorers + + self.create = to_raw_response_wrapper( + scorers.create, + ) + self.retrieve = to_raw_response_wrapper( + scorers.retrieve, + ) + self.update = to_raw_response_wrapper( + scorers.update, + ) + self.list = to_raw_response_wrapper( + scorers.list, + ) + self.validate = to_raw_response_wrapper( + scorers.validate, + ) + + +class AsyncScorersResourceWithRawResponse: + def __init__(self, scorers: AsyncScorersResource) -> None: + self._scorers = scorers + + self.create = async_to_raw_response_wrapper( + scorers.create, + ) + self.retrieve = async_to_raw_response_wrapper( + scorers.retrieve, + ) + self.update = async_to_raw_response_wrapper( + scorers.update, + ) + self.list = async_to_raw_response_wrapper( + scorers.list, + ) + self.validate = async_to_raw_response_wrapper( + scorers.validate, + ) + + +class ScorersResourceWithStreamingResponse: + def __init__(self, scorers: ScorersResource) -> None: + self._scorers = scorers + + self.create = to_streamed_response_wrapper( + scorers.create, + ) + self.retrieve = to_streamed_response_wrapper( + scorers.retrieve, + ) + self.update = to_streamed_response_wrapper( + scorers.update, + ) + self.list = to_streamed_response_wrapper( + scorers.list, + ) + self.validate = to_streamed_response_wrapper( + scorers.validate, + ) + + +class AsyncScorersResourceWithStreamingResponse: + def __init__(self, scorers: AsyncScorersResource) -> None: + self._scorers = scorers + + self.create = async_to_streamed_response_wrapper( + scorers.create, + ) + self.retrieve = async_to_streamed_response_wrapper( + scorers.retrieve, + ) + self.update = async_to_streamed_response_wrapper( + scorers.update, + ) + self.list = async_to_streamed_response_wrapper( + scorers.list, + ) + self.validate = async_to_streamed_response_wrapper( + scorers.validate, + ) diff --git a/src/runloop_api_client/types/input_context.py b/src/runloop_api_client/types/input_context.py index 0ca7afafb..5cc697db9 100644 --- a/src/runloop_api_client/types/input_context.py +++ b/src/runloop_api_client/types/input_context.py @@ -1,5 +1,6 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. +from typing import Optional from .._models import BaseModel @@ -9,3 +10,6 @@ class InputContext(BaseModel): problem_statement: str """The problem statement for the Scenario.""" + + additional_context: Optional[object] = None + """Additional JSON structured input context.""" diff --git a/src/runloop_api_client/types/input_context_param.py b/src/runloop_api_client/types/input_context_param.py index dcea87fa2..7f977ad65 100644 --- a/src/runloop_api_client/types/input_context_param.py +++ b/src/runloop_api_client/types/input_context_param.py @@ -2,6 +2,7 @@ from __future__ import annotations +from typing import Optional from typing_extensions import Required, TypedDict __all__ = ["InputContextParam"] @@ -10,3 +11,6 @@ class InputContextParam(TypedDict, total=False): problem_statement: Required[str] """The problem statement for the Scenario.""" + + additional_context: Optional[object] + """Additional JSON structured input context.""" diff --git a/src/runloop_api_client/types/scenarios/__init__.py b/src/runloop_api_client/types/scenarios/__init__.py index 4bc4e1112..d25c85c4e 100644 --- a/src/runloop_api_client/types/scenarios/__init__.py +++ b/src/runloop_api_client/types/scenarios/__init__.py @@ -3,3 +3,12 @@ from __future__ import annotations from .run_list_params import RunListParams as RunListParams +from .scorer_list_params import ScorerListParams as ScorerListParams +from .scorer_create_params import ScorerCreateParams as ScorerCreateParams +from .scorer_list_response import ScorerListResponse as ScorerListResponse +from .scorer_update_params import ScorerUpdateParams as ScorerUpdateParams +from .scorer_create_response import ScorerCreateResponse as ScorerCreateResponse +from .scorer_update_response import ScorerUpdateResponse as ScorerUpdateResponse +from .scorer_validate_params import ScorerValidateParams as ScorerValidateParams +from .scorer_retrieve_response import ScorerRetrieveResponse as ScorerRetrieveResponse +from .scorer_validate_response import ScorerValidateResponse as ScorerValidateResponse diff --git a/src/runloop_api_client/types/scenarios/scorer_create_params.py b/src/runloop_api_client/types/scenarios/scorer_create_params.py new file mode 100644 index 000000000..8261dff54 --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_create_params.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, TypedDict + +__all__ = ["ScorerCreateParams"] + + +class ScorerCreateParams(TypedDict, total=False): + bash_script: Required[str] + """ + Bash script for the custom scorer taking context as a json object + $RL_TEST_CONTEXT. + """ + + name: Required[str] + """Name of the custom scorer.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_create_response.py b/src/runloop_api_client/types/scenarios/scorer_create_response.py new file mode 100644 index 000000000..8c9adf293 --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_create_response.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + + +from ..._models import BaseModel + +__all__ = ["ScorerCreateResponse"] + + +class ScorerCreateResponse(BaseModel): + id: str + """ID for the scenario scorer.""" + + bash_script: str + """Bash script that takes in $RL_TEST_CONTEXT as env variable and runs scoring.""" + + name: str + """Name of the scenario scorer.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_list_params.py b/src/runloop_api_client/types/scenarios/scorer_list_params.py new file mode 100644 index 000000000..0577a327e --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_list_params.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +__all__ = ["ScorerListParams"] + + +class ScorerListParams(TypedDict, total=False): + limit: int + """The limit of items to return. Default is 20.""" + + starting_after: str + """Load the next page of data starting after the item with the given ID.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_list_response.py b/src/runloop_api_client/types/scenarios/scorer_list_response.py new file mode 100644 index 000000000..59f7adc40 --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_list_response.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + + +from ..._models import BaseModel + +__all__ = ["ScorerListResponse"] + + +class ScorerListResponse(BaseModel): + id: str + """ID for the scenario scorer.""" + + bash_script: str + """Bash script that takes in $RL_TEST_CONTEXT as env variable and runs scoring.""" + + name: str + """Name of the scenario scorer.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_retrieve_response.py b/src/runloop_api_client/types/scenarios/scorer_retrieve_response.py new file mode 100644 index 000000000..25ab5b1d6 --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_retrieve_response.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + + +from ..._models import BaseModel + +__all__ = ["ScorerRetrieveResponse"] + + +class ScorerRetrieveResponse(BaseModel): + id: str + """ID for the scenario scorer.""" + + bash_script: str + """Bash script that takes in $RL_TEST_CONTEXT as env variable and runs scoring.""" + + name: str + """Name of the scenario scorer.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_update_params.py b/src/runloop_api_client/types/scenarios/scorer_update_params.py new file mode 100644 index 000000000..bff2e81c0 --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_update_params.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, TypedDict + +__all__ = ["ScorerUpdateParams"] + + +class ScorerUpdateParams(TypedDict, total=False): + bash_script: Required[str] + """ + Bash script for the custom scorer taking context as a json object + $RL_TEST_CONTEXT. + """ + + name: Required[str] + """Name of the custom scorer.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_update_response.py b/src/runloop_api_client/types/scenarios/scorer_update_response.py new file mode 100644 index 000000000..72c6dce78 --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_update_response.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + + +from ..._models import BaseModel + +__all__ = ["ScorerUpdateResponse"] + + +class ScorerUpdateResponse(BaseModel): + id: str + """ID for the scenario scorer.""" + + bash_script: str + """Bash script that takes in $RL_TEST_CONTEXT as env variable and runs scoring.""" + + name: str + """Name of the scenario scorer.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_validate_params.py b/src/runloop_api_client/types/scenarios/scorer_validate_params.py new file mode 100644 index 000000000..41215ea2d --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_validate_params.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, TypedDict + +from ..scenario_environment_param import ScenarioEnvironmentParam + +__all__ = ["ScorerValidateParams"] + + +class ScorerValidateParams(TypedDict, total=False): + scoring_context: Required[object] + """Json context that gets passed to the custom scorer""" + + environment_parameters: ScenarioEnvironmentParam + """The Environment in which the Scenario will run.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_validate_response.py b/src/runloop_api_client/types/scenarios/scorer_validate_response.py new file mode 100644 index 000000000..6c2755e6c --- /dev/null +++ b/src/runloop_api_client/types/scenarios/scorer_validate_response.py @@ -0,0 +1,23 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel +from ..scenario_environment import ScenarioEnvironment +from ..scoring_function_result_view import ScoringFunctionResultView + +__all__ = ["ScorerValidateResponse"] + + +class ScorerValidateResponse(BaseModel): + name: str + """Name of the custom scorer.""" + + scoring_context: object + """Json context that gets passed to the custom scorer""" + + scoring_result: ScoringFunctionResultView + """Result of the scoring function.""" + + environment_parameters: Optional[ScenarioEnvironment] = None + """The Environment in which the Scenario will run.""" diff --git a/tests/api_resources/scenarios/test_scorers.py b/tests/api_resources/scenarios/test_scorers.py new file mode 100644 index 000000000..aa0d641e3 --- /dev/null +++ b/tests/api_resources/scenarios/test_scorers.py @@ -0,0 +1,441 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from tests.utils import assert_matches_type +from runloop_api_client import Runloop, AsyncRunloop +from runloop_api_client.pagination import SyncScenarioScorersCursorIDPage, AsyncScenarioScorersCursorIDPage +from runloop_api_client.types.scenarios import ( + ScorerListResponse, + ScorerCreateResponse, + ScorerUpdateResponse, + ScorerRetrieveResponse, + ScorerValidateResponse, +) + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestScorers: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + def test_method_create(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.create( + bash_script="bash_script", + name="name", + ) + assert_matches_type(ScorerCreateResponse, scorer, path=["response"]) + + @parametrize + def test_raw_response_create(self, client: Runloop) -> None: + response = client.scenarios.scorers.with_raw_response.create( + bash_script="bash_script", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = response.parse() + assert_matches_type(ScorerCreateResponse, scorer, path=["response"]) + + @parametrize + def test_streaming_response_create(self, client: Runloop) -> None: + with client.scenarios.scorers.with_streaming_response.create( + bash_script="bash_script", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = response.parse() + assert_matches_type(ScorerCreateResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_retrieve(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.retrieve( + "id", + ) + assert_matches_type(ScorerRetrieveResponse, scorer, path=["response"]) + + @parametrize + def test_raw_response_retrieve(self, client: Runloop) -> None: + response = client.scenarios.scorers.with_raw_response.retrieve( + "id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = response.parse() + assert_matches_type(ScorerRetrieveResponse, scorer, path=["response"]) + + @parametrize + def test_streaming_response_retrieve(self, client: Runloop) -> None: + with client.scenarios.scorers.with_streaming_response.retrieve( + "id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = response.parse() + assert_matches_type(ScorerRetrieveResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_retrieve(self, client: Runloop) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + client.scenarios.scorers.with_raw_response.retrieve( + "", + ) + + @parametrize + def test_method_update(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.update( + id="id", + bash_script="bash_script", + name="name", + ) + assert_matches_type(ScorerUpdateResponse, scorer, path=["response"]) + + @parametrize + def test_raw_response_update(self, client: Runloop) -> None: + response = client.scenarios.scorers.with_raw_response.update( + id="id", + bash_script="bash_script", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = response.parse() + assert_matches_type(ScorerUpdateResponse, scorer, path=["response"]) + + @parametrize + def test_streaming_response_update(self, client: Runloop) -> None: + with client.scenarios.scorers.with_streaming_response.update( + id="id", + bash_script="bash_script", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = response.parse() + assert_matches_type(ScorerUpdateResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_update(self, client: Runloop) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + client.scenarios.scorers.with_raw_response.update( + id="", + bash_script="bash_script", + name="name", + ) + + @parametrize + def test_method_list(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.list() + assert_matches_type(SyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + @parametrize + def test_method_list_with_all_params(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.list( + limit=0, + starting_after="starting_after", + ) + assert_matches_type(SyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + @parametrize + def test_raw_response_list(self, client: Runloop) -> None: + response = client.scenarios.scorers.with_raw_response.list() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = response.parse() + assert_matches_type(SyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + @parametrize + def test_streaming_response_list(self, client: Runloop) -> None: + with client.scenarios.scorers.with_streaming_response.list() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = response.parse() + assert_matches_type(SyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_validate(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.validate( + id="id", + scoring_context={}, + ) + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + @parametrize + def test_method_validate_with_all_params(self, client: Runloop) -> None: + scorer = client.scenarios.scorers.validate( + id="id", + scoring_context={}, + environment_parameters={ + "blueprint_id": "blueprint_id", + "prebuilt_id": "prebuilt_id", + "snapshot_id": "snapshot_id", + }, + ) + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + @parametrize + def test_raw_response_validate(self, client: Runloop) -> None: + response = client.scenarios.scorers.with_raw_response.validate( + id="id", + scoring_context={}, + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = response.parse() + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + @parametrize + def test_streaming_response_validate(self, client: Runloop) -> None: + with client.scenarios.scorers.with_streaming_response.validate( + id="id", + scoring_context={}, + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = response.parse() + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_validate(self, client: Runloop) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + client.scenarios.scorers.with_raw_response.validate( + id="", + scoring_context={}, + ) + + +class TestAsyncScorers: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + async def test_method_create(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.create( + bash_script="bash_script", + name="name", + ) + assert_matches_type(ScorerCreateResponse, scorer, path=["response"]) + + @parametrize + async def test_raw_response_create(self, async_client: AsyncRunloop) -> None: + response = await async_client.scenarios.scorers.with_raw_response.create( + bash_script="bash_script", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = await response.parse() + assert_matches_type(ScorerCreateResponse, scorer, path=["response"]) + + @parametrize + async def test_streaming_response_create(self, async_client: AsyncRunloop) -> None: + async with async_client.scenarios.scorers.with_streaming_response.create( + bash_script="bash_script", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = await response.parse() + assert_matches_type(ScorerCreateResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_retrieve(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.retrieve( + "id", + ) + assert_matches_type(ScorerRetrieveResponse, scorer, path=["response"]) + + @parametrize + async def test_raw_response_retrieve(self, async_client: AsyncRunloop) -> None: + response = await async_client.scenarios.scorers.with_raw_response.retrieve( + "id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = await response.parse() + assert_matches_type(ScorerRetrieveResponse, scorer, path=["response"]) + + @parametrize + async def test_streaming_response_retrieve(self, async_client: AsyncRunloop) -> None: + async with async_client.scenarios.scorers.with_streaming_response.retrieve( + "id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = await response.parse() + assert_matches_type(ScorerRetrieveResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_retrieve(self, async_client: AsyncRunloop) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + await async_client.scenarios.scorers.with_raw_response.retrieve( + "", + ) + + @parametrize + async def test_method_update(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.update( + id="id", + bash_script="bash_script", + name="name", + ) + assert_matches_type(ScorerUpdateResponse, scorer, path=["response"]) + + @parametrize + async def test_raw_response_update(self, async_client: AsyncRunloop) -> None: + response = await async_client.scenarios.scorers.with_raw_response.update( + id="id", + bash_script="bash_script", + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = await response.parse() + assert_matches_type(ScorerUpdateResponse, scorer, path=["response"]) + + @parametrize + async def test_streaming_response_update(self, async_client: AsyncRunloop) -> None: + async with async_client.scenarios.scorers.with_streaming_response.update( + id="id", + bash_script="bash_script", + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = await response.parse() + assert_matches_type(ScorerUpdateResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_update(self, async_client: AsyncRunloop) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + await async_client.scenarios.scorers.with_raw_response.update( + id="", + bash_script="bash_script", + name="name", + ) + + @parametrize + async def test_method_list(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.list() + assert_matches_type(AsyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + @parametrize + async def test_method_list_with_all_params(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.list( + limit=0, + starting_after="starting_after", + ) + assert_matches_type(AsyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + @parametrize + async def test_raw_response_list(self, async_client: AsyncRunloop) -> None: + response = await async_client.scenarios.scorers.with_raw_response.list() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = await response.parse() + assert_matches_type(AsyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + @parametrize + async def test_streaming_response_list(self, async_client: AsyncRunloop) -> None: + async with async_client.scenarios.scorers.with_streaming_response.list() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = await response.parse() + assert_matches_type(AsyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_validate(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.validate( + id="id", + scoring_context={}, + ) + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + @parametrize + async def test_method_validate_with_all_params(self, async_client: AsyncRunloop) -> None: + scorer = await async_client.scenarios.scorers.validate( + id="id", + scoring_context={}, + environment_parameters={ + "blueprint_id": "blueprint_id", + "prebuilt_id": "prebuilt_id", + "snapshot_id": "snapshot_id", + }, + ) + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + @parametrize + async def test_raw_response_validate(self, async_client: AsyncRunloop) -> None: + response = await async_client.scenarios.scorers.with_raw_response.validate( + id="id", + scoring_context={}, + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + scorer = await response.parse() + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + @parametrize + async def test_streaming_response_validate(self, async_client: AsyncRunloop) -> None: + async with async_client.scenarios.scorers.with_streaming_response.validate( + id="id", + scoring_context={}, + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + scorer = await response.parse() + assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_validate(self, async_client: AsyncRunloop) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + await async_client.scenarios.scorers.with_raw_response.validate( + id="", + scoring_context={}, + ) diff --git a/tests/api_resources/test_scenarios.py b/tests/api_resources/test_scenarios.py index e204eb126..64ae41c99 100644 --- a/tests/api_resources/test_scenarios.py +++ b/tests/api_resources/test_scenarios.py @@ -40,7 +40,10 @@ def test_method_create(self, client: Runloop) -> None: @parametrize def test_method_create_with_all_params(self, client: Runloop) -> None: scenario = client.scenarios.create( - input_context={"problem_statement": "problem_statement"}, + input_context={ + "problem_statement": "problem_statement", + "additional_context": {}, + }, name="name", scoring_contract={ "scoring_function_parameters": [ @@ -270,7 +273,10 @@ async def test_method_create(self, async_client: AsyncRunloop) -> None: @parametrize async def test_method_create_with_all_params(self, async_client: AsyncRunloop) -> None: scenario = await async_client.scenarios.create( - input_context={"problem_statement": "problem_statement"}, + input_context={ + "problem_statement": "problem_statement", + "additional_context": {}, + }, name="name", scoring_contract={ "scoring_function_parameters": [ From 6933de364b487b46699813787a2c325389bebdc7 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 18:32:16 +0000 Subject: [PATCH 4/4] release: 0.20.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 15 +++++++++++++++ pyproject.toml | 2 +- src/runloop_api_client/_version.py | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index e75629345..0c2ecec68 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.19.0" + ".": "0.20.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f53ae99c6..bc0f66314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 0.20.0 (2025-02-04) + +Full Changelog: [v0.19.0...v0.20.0](https://github.com/runloopai/api-client-python/compare/v0.19.0...v0.20.0) + +### Features + +* add helpers for blueprint and scenario run creation ([b00a7c1](https://github.com/runloopai/api-client-python/commit/b00a7c1b935db7c919d79ed10c135bf5ed2a9b4f)) +* **api:** api update ([#526](https://github.com/runloopai/api-client-python/issues/526)) ([be26ed7](https://github.com/runloopai/api-client-python/commit/be26ed7c82017d9bff67c9920c078808f8a675e3)) + + +### Chores + +* **internal:** bummp ruff dependency ([#525](https://github.com/runloopai/api-client-python/issues/525)) ([84051db](https://github.com/runloopai/api-client-python/commit/84051db83a6347f28ee4891be52f5eee7fa9d02a)) +* **internal:** change default timeout to an int ([#524](https://github.com/runloopai/api-client-python/issues/524)) ([a15ccaa](https://github.com/runloopai/api-client-python/commit/a15ccaa45f69331d57be0144a16ba3b1df387245)) + ## 0.19.0 (2025-02-03) Full Changelog: [v0.18.0...v0.19.0](https://github.com/runloopai/api-client-python/compare/v0.18.0...v0.19.0) diff --git a/pyproject.toml b/pyproject.toml index fd789f00c..84809bd4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "runloop_api_client" -version = "0.19.0" +version = "0.20.0" description = "The official Python library for the runloop API" dynamic = ["readme"] license = "MIT" diff --git a/src/runloop_api_client/_version.py b/src/runloop_api_client/_version.py index 5b3613c63..8ec13f75d 100644 --- a/src/runloop_api_client/_version.py +++ b/src/runloop_api_client/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "runloop_api_client" -__version__ = "0.19.0" # x-release-please-version +__version__ = "0.20.0" # x-release-please-version