diff --git a/.stats.yml b/.stats.yml
index 89dc8775..08c183bf 100644
--- a/.stats.yml
+++ b/.stats.yml
@@ -1,2 +1,2 @@
-configured_endpoints: 77
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-cb8add05a7b418d6f8a5624be8477564853da49e8bf9671ae89b8ce49a04b6cd.yml
+configured_endpoints: 78
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-4ccbc7c04012cbcca678f13e39f66bb770b8b3a9d6f1815ce1b9c20fee099128.yml
diff --git a/api.md b/api.md
index 90c23734..aaecbbfd 100644
--- a/api.md
+++ b/api.md
@@ -278,6 +278,7 @@ Methods:
- client.scenarios.create(\*\*params) -> ScenarioView
- client.scenarios.retrieve(id) -> ScenarioView
+- client.scenarios.update(id, \*\*params) -> ScenarioView
- client.scenarios.list(\*\*params) -> SyncScenariosCursorIDPage[ScenarioView]
- client.scenarios.list_public(\*\*params) -> SyncScenariosCursorIDPage[ScenarioView]
- client.scenarios.start_run(\*\*params) -> ScenarioRunView
diff --git a/src/runloop_api_client/resources/scenarios/scenarios.py b/src/runloop_api_client/resources/scenarios/scenarios.py
index 92702801..b42f56b1 100644
--- a/src/runloop_api_client/resources/scenarios/scenarios.py
+++ b/src/runloop_api_client/resources/scenarios/scenarios.py
@@ -17,6 +17,7 @@
from ...types import (
scenario_list_params,
scenario_create_params,
+ scenario_update_params,
scenario_start_run_params,
scenario_list_public_params,
)
@@ -182,6 +183,78 @@ def retrieve(
cast_to=ScenarioView,
)
+ def update(
+ self,
+ id: str,
+ *,
+ input_context: InputContextParam,
+ name: str,
+ scoring_contract: ScoringContractParam,
+ environment_parameters: Optional[ScenarioEnvironmentParam] | NotGiven = NOT_GIVEN,
+ metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
+ reference_output: Optional[str] | NotGiven = NOT_GIVEN,
+ # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+ # The extra values given here take precedence over values defined on the client or passed to this method.
+ extra_headers: Headers | None = None,
+ extra_query: Query | None = None,
+ extra_body: Body | None = None,
+ timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+ idempotency_key: str | None = None,
+ ) -> ScenarioView:
+ """
+ Update a Scenario, a repeatable AI coding evaluation test that defines the
+ starting environment as well as evaluation success criteria.
+
+ Args:
+ input_context: The input context for the Scenario.
+
+ name: Name of the scenario.
+
+ scoring_contract: The scoring contract for the Scenario.
+
+ environment_parameters: The Environment in which the Scenario will run.
+
+ metadata: User defined metadata to attach to the scenario for organization.
+
+ reference_output: A string representation of the reference output to solve the scenario. Commonly
+ can be the result of a git diff or a sequence of command actions to apply to the
+ environment.
+
+ extra_headers: Send extra headers
+
+ extra_query: Add additional query parameters to the request
+
+ extra_body: Add additional JSON properties to the request
+
+ timeout: Override the client-level default timeout for this request, in seconds
+
+ idempotency_key: Specify a custom idempotency key for this request
+ """
+ if not id:
+ raise ValueError(f"Expected a non-empty value for `id` but received {id!r}")
+ return self._post(
+ f"/v1/scenarios/{id}",
+ body=maybe_transform(
+ {
+ "input_context": input_context,
+ "name": name,
+ "scoring_contract": scoring_contract,
+ "environment_parameters": environment_parameters,
+ "metadata": metadata,
+ "reference_output": reference_output,
+ },
+ scenario_update_params.ScenarioUpdateParams,
+ ),
+ options=make_request_options(
+ extra_headers=extra_headers,
+ extra_query=extra_query,
+ extra_body=extra_body,
+ timeout=timeout,
+ idempotency_key=idempotency_key,
+ ),
+ cast_to=ScenarioView,
+ )
+
def list(
self,
*,
@@ -474,6 +547,78 @@ async def retrieve(
cast_to=ScenarioView,
)
+ async def update(
+ self,
+ id: str,
+ *,
+ input_context: InputContextParam,
+ name: str,
+ scoring_contract: ScoringContractParam,
+ environment_parameters: Optional[ScenarioEnvironmentParam] | NotGiven = NOT_GIVEN,
+ metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
+ reference_output: Optional[str] | NotGiven = NOT_GIVEN,
+ # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+ # The extra values given here take precedence over values defined on the client or passed to this method.
+ extra_headers: Headers | None = None,
+ extra_query: Query | None = None,
+ extra_body: Body | None = None,
+ timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+ idempotency_key: str | None = None,
+ ) -> ScenarioView:
+ """
+ Update a Scenario, a repeatable AI coding evaluation test that defines the
+ starting environment as well as evaluation success criteria.
+
+ Args:
+ input_context: The input context for the Scenario.
+
+ name: Name of the scenario.
+
+ scoring_contract: The scoring contract for the Scenario.
+
+ environment_parameters: The Environment in which the Scenario will run.
+
+ metadata: User defined metadata to attach to the scenario for organization.
+
+ reference_output: A string representation of the reference output to solve the scenario. Commonly
+ can be the result of a git diff or a sequence of command actions to apply to the
+ environment.
+
+ extra_headers: Send extra headers
+
+ extra_query: Add additional query parameters to the request
+
+ extra_body: Add additional JSON properties to the request
+
+ timeout: Override the client-level default timeout for this request, in seconds
+
+ idempotency_key: Specify a custom idempotency key for this request
+ """
+ if not id:
+ raise ValueError(f"Expected a non-empty value for `id` but received {id!r}")
+ return await self._post(
+ f"/v1/scenarios/{id}",
+ body=await async_maybe_transform(
+ {
+ "input_context": input_context,
+ "name": name,
+ "scoring_contract": scoring_contract,
+ "environment_parameters": environment_parameters,
+ "metadata": metadata,
+ "reference_output": reference_output,
+ },
+ scenario_update_params.ScenarioUpdateParams,
+ ),
+ options=make_request_options(
+ extra_headers=extra_headers,
+ extra_query=extra_query,
+ extra_body=extra_body,
+ timeout=timeout,
+ idempotency_key=idempotency_key,
+ ),
+ cast_to=ScenarioView,
+ )
+
def list(
self,
*,
@@ -646,6 +791,9 @@ def __init__(self, scenarios: ScenariosResource) -> None:
self.retrieve = to_raw_response_wrapper(
scenarios.retrieve,
)
+ self.update = to_raw_response_wrapper(
+ scenarios.update,
+ )
self.list = to_raw_response_wrapper(
scenarios.list,
)
@@ -675,6 +823,9 @@ def __init__(self, scenarios: AsyncScenariosResource) -> None:
self.retrieve = async_to_raw_response_wrapper(
scenarios.retrieve,
)
+ self.update = async_to_raw_response_wrapper(
+ scenarios.update,
+ )
self.list = async_to_raw_response_wrapper(
scenarios.list,
)
@@ -704,6 +855,9 @@ def __init__(self, scenarios: ScenariosResource) -> None:
self.retrieve = to_streamed_response_wrapper(
scenarios.retrieve,
)
+ self.update = to_streamed_response_wrapper(
+ scenarios.update,
+ )
self.list = to_streamed_response_wrapper(
scenarios.list,
)
@@ -733,6 +887,9 @@ def __init__(self, scenarios: AsyncScenariosResource) -> None:
self.retrieve = async_to_streamed_response_wrapper(
scenarios.retrieve,
)
+ self.update = async_to_streamed_response_wrapper(
+ scenarios.update,
+ )
self.list = async_to_streamed_response_wrapper(
scenarios.list,
)
diff --git a/src/runloop_api_client/types/__init__.py b/src/runloop_api_client/types/__init__.py
index 62b6d30e..31b76176 100644
--- a/src/runloop_api_client/types/__init__.py
+++ b/src/runloop_api_client/types/__init__.py
@@ -35,6 +35,7 @@
from .repository_list_params import RepositoryListParams as RepositoryListParams
from .scenario_create_params import ScenarioCreateParams as ScenarioCreateParams
from .scenario_run_list_view import ScenarioRunListView as ScenarioRunListView
+from .scenario_update_params import ScenarioUpdateParams as ScenarioUpdateParams
from .scoring_contract_param import ScoringContractParam as ScoringContractParam
from .scoring_function_param import ScoringFunctionParam as ScoringFunctionParam
from .benchmark_create_params import BenchmarkCreateParams as BenchmarkCreateParams
diff --git a/src/runloop_api_client/types/scenario_update_params.py b/src/runloop_api_client/types/scenario_update_params.py
new file mode 100644
index 00000000..1004e7bc
--- /dev/null
+++ b/src/runloop_api_client/types/scenario_update_params.py
@@ -0,0 +1,36 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, Optional
+from typing_extensions import Required, TypedDict
+
+from .input_context_param import InputContextParam
+from .scoring_contract_param import ScoringContractParam
+from .scenario_environment_param import ScenarioEnvironmentParam
+
+__all__ = ["ScenarioUpdateParams"]
+
+
+class ScenarioUpdateParams(TypedDict, total=False):
+ input_context: Required[InputContextParam]
+ """The input context for the Scenario."""
+
+ name: Required[str]
+ """Name of the scenario."""
+
+ scoring_contract: Required[ScoringContractParam]
+ """The scoring contract for the Scenario."""
+
+ environment_parameters: Optional[ScenarioEnvironmentParam]
+ """The Environment in which the Scenario will run."""
+
+ metadata: Optional[Dict[str, str]]
+ """User defined metadata to attach to the scenario for organization."""
+
+ reference_output: Optional[str]
+ """A string representation of the reference output to solve the scenario.
+
+ Commonly can be the result of a git diff or a sequence of command actions to
+ apply to the environment.
+ """
diff --git a/src/runloop_api_client/types/scoring_function.py b/src/runloop_api_client/types/scoring_function.py
index bef775ec..f7a2a6e0 100644
--- a/src/runloop_api_client/types/scoring_function.py
+++ b/src/runloop_api_client/types/scoring_function.py
@@ -20,7 +20,7 @@ class ScoringFunction(BaseModel):
"""
weight: float
- """Wight to apply to scoring function score.
+ """Weight to apply to scoring function score.
Weights of all scoring functions should sum to 1.0.
"""
diff --git a/src/runloop_api_client/types/scoring_function_param.py b/src/runloop_api_client/types/scoring_function_param.py
index 0caa6146..5bcbacc5 100644
--- a/src/runloop_api_client/types/scoring_function_param.py
+++ b/src/runloop_api_client/types/scoring_function_param.py
@@ -21,7 +21,7 @@ class ScoringFunctionParam(TypedDict, total=False):
"""
weight: Required[float]
- """Wight to apply to scoring function score.
+ """Weight to apply to scoring function score.
Weights of all scoring functions should sum to 1.0.
"""
diff --git a/tests/api_resources/test_scenarios.py b/tests/api_resources/test_scenarios.py
index 245da96b..218ebb54 100644
--- a/tests/api_resources/test_scenarios.py
+++ b/tests/api_resources/test_scenarios.py
@@ -160,6 +160,129 @@ def test_path_params_retrieve(self, client: Runloop) -> None:
"",
)
+ @parametrize
+ def test_method_update(self, client: Runloop) -> None:
+ scenario = client.scenarios.update(
+ id="id",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ )
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ @parametrize
+ def test_method_update_with_all_params(self, client: Runloop) -> None:
+ scenario = client.scenarios.update(
+ id="id",
+ input_context={
+ "problem_statement": "problem_statement",
+ "additional_context": {},
+ },
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ "bash_script": "bash_script",
+ "scorer_params": {},
+ }
+ ]
+ },
+ environment_parameters={
+ "blueprint_id": "blueprint_id",
+ "launch_parameters": {
+ "after_idle": {
+ "idle_time_seconds": 0,
+ "on_idle": "shutdown",
+ },
+ "available_ports": [0],
+ "keep_alive_time_seconds": 0,
+ "launch_commands": ["string"],
+ "resource_size_request": "SMALL",
+ },
+ "prebuilt_id": "prebuilt_id",
+ "snapshot_id": "snapshot_id",
+ "working_directory": "working_directory",
+ },
+ metadata={"foo": "string"},
+ reference_output="reference_output",
+ )
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ @parametrize
+ def test_raw_response_update(self, client: Runloop) -> None:
+ response = client.scenarios.with_raw_response.update(
+ id="id",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ )
+
+ assert response.is_closed is True
+ assert response.http_request.headers.get("X-Stainless-Lang") == "python"
+ scenario = response.parse()
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ @parametrize
+ def test_streaming_response_update(self, client: Runloop) -> None:
+ with client.scenarios.with_streaming_response.update(
+ id="id",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ ) as response:
+ assert not response.is_closed
+ assert response.http_request.headers.get("X-Stainless-Lang") == "python"
+
+ scenario = response.parse()
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ assert cast(Any, response.is_closed) is True
+
+ @parametrize
+ def test_path_params_update(self, client: Runloop) -> None:
+ with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"):
+ client.scenarios.with_raw_response.update(
+ id="",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ )
+
@parametrize
def test_method_list(self, client: Runloop) -> None:
scenario = client.scenarios.list()
@@ -412,6 +535,129 @@ async def test_path_params_retrieve(self, async_client: AsyncRunloop) -> None:
"",
)
+ @parametrize
+ async def test_method_update(self, async_client: AsyncRunloop) -> None:
+ scenario = await async_client.scenarios.update(
+ id="id",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ )
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ @parametrize
+ async def test_method_update_with_all_params(self, async_client: AsyncRunloop) -> None:
+ scenario = await async_client.scenarios.update(
+ id="id",
+ input_context={
+ "problem_statement": "problem_statement",
+ "additional_context": {},
+ },
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ "bash_script": "bash_script",
+ "scorer_params": {},
+ }
+ ]
+ },
+ environment_parameters={
+ "blueprint_id": "blueprint_id",
+ "launch_parameters": {
+ "after_idle": {
+ "idle_time_seconds": 0,
+ "on_idle": "shutdown",
+ },
+ "available_ports": [0],
+ "keep_alive_time_seconds": 0,
+ "launch_commands": ["string"],
+ "resource_size_request": "SMALL",
+ },
+ "prebuilt_id": "prebuilt_id",
+ "snapshot_id": "snapshot_id",
+ "working_directory": "working_directory",
+ },
+ metadata={"foo": "string"},
+ reference_output="reference_output",
+ )
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ @parametrize
+ async def test_raw_response_update(self, async_client: AsyncRunloop) -> None:
+ response = await async_client.scenarios.with_raw_response.update(
+ id="id",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ )
+
+ assert response.is_closed is True
+ assert response.http_request.headers.get("X-Stainless-Lang") == "python"
+ scenario = await response.parse()
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ @parametrize
+ async def test_streaming_response_update(self, async_client: AsyncRunloop) -> None:
+ async with async_client.scenarios.with_streaming_response.update(
+ id="id",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ ) as response:
+ assert not response.is_closed
+ assert response.http_request.headers.get("X-Stainless-Lang") == "python"
+
+ scenario = await response.parse()
+ assert_matches_type(ScenarioView, scenario, path=["response"])
+
+ assert cast(Any, response.is_closed) is True
+
+ @parametrize
+ async def test_path_params_update(self, async_client: AsyncRunloop) -> None:
+ with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"):
+ await async_client.scenarios.with_raw_response.update(
+ id="",
+ input_context={"problem_statement": "problem_statement"},
+ name="name",
+ scoring_contract={
+ "scoring_function_parameters": [
+ {
+ "name": "name",
+ "type": "type",
+ "weight": 0,
+ }
+ ]
+ },
+ )
+
@parametrize
async def test_method_list(self, async_client: AsyncRunloop) -> None:
scenario = await async_client.scenarios.list()