From 665fae85a75c301a3d23219a4c491f8f7f080f67 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Thu, 7 Nov 2024 22:44:55 -0800 Subject: [PATCH 1/3] evals api --- src/llama_stack_client/_client.py | 32 +- src/llama_stack_client/_compat.py | 6 +- src/llama_stack_client/_models.py | 9 +- src/llama_stack_client/_utils/__init__.py | 1 + src/llama_stack_client/_utils/_transform.py | 9 +- src/llama_stack_client/_utils/_utils.py | 17 + src/llama_stack_client/resources/__init__.py | 24 +- .../resources/agents/agents.py | 5 +- .../resources/agents/turn.py | 80 ++-- .../resources/eval/__init__.py | 28 +- src/llama_stack_client/resources/eval/eval.py | 148 +++---- src/llama_stack_client/resources/eval/jobs.py | 396 +++++++++++++++++ src/llama_stack_client/resources/safety.py | 8 +- src/llama_stack_client/resources/scoring.py | 30 +- src/llama_stack_client/resources/shields.py | 8 +- src/llama_stack_client/types/__init__.py | 21 +- .../types/agent_create_params.py | 222 +--------- .../types/agents/turn_create_response.py | 70 +-- .../batch_inference_chat_completion_params.py | 15 +- .../batch_inference_completion_params.py | 6 +- .../types/dataset_list_response.py | 75 +++- .../types/dataset_register_params.py | 76 +++- .../types/dataset_retrieve_response.py | 75 +++- src/llama_stack_client/types/eval/__init__.py | 6 +- .../types/eval/job_cancel_params.py | 2 + .../types/eval/job_retrieve_params.py | 17 + .../types/eval/job_retrieve_response.py | 14 + .../types/eval/job_status_params.py | 2 + .../types/eval/job_status_response.py | 8 + .../types/eval_evaluate_rows_params.py | 125 ++++++ .../types/eval_evaluate_rows_response.py | 14 + .../types/eval_run_eval_params.py | 121 +++++ .../types/eval_run_eval_response.py | 10 + .../types/inference_chat_completion_params.py | 24 +- .../types/inference_completion_params.py | 16 +- .../types/inference_embeddings_params.py | 6 +- .../types/memory_insert_params.py | 6 +- .../types/memory_query_params.py | 6 +- .../types/memory_retrieval_step.py | 6 +- .../types/model_def_with_provider.py | 3 + .../types/model_def_with_provider_param.py | 4 +- .../types/query_documents_response.py | 6 +- .../types/safety_run_shield_params.py | 4 +- .../types/scoring_fn_def_with_provider.py | 54 +-- .../scoring_fn_def_with_provider_param.py | 54 +-- .../types/scoring_score_batch_params.py | 34 +- .../types/scoring_score_batch_response.py | 14 + .../types/scoring_score_params.py | 34 +- .../types/scoring_score_response.py | 12 + .../types/shared/__init__.py | 9 + .../types/shared/agent_config.py | 57 +++ .../types/shared/attachment.py | 6 +- .../code_interpreter_tool_definition.py | 21 + .../types/shared/completion_message.py | 6 +- .../shared/function_call_tool_definition.py | 26 ++ .../types/shared/memory_tool_definition.py | 91 ++++ .../types/shared/photogen_tool_definition.py | 19 + .../types/shared/scoring_result.py | 13 + .../types/shared/search_tool_definition.py | 23 + .../types/shared/system_message.py | 6 +- .../types/shared/tool_param_definition.py | 17 + .../types/shared/tool_response_message.py | 6 +- .../types/shared/user_message.py | 10 +- .../shared/wolfram_alpha_tool_definition.py | 21 + .../types/shared_params/__init__.py | 8 + .../types/shared_params/agent_config.py | 58 +++ .../types/shared_params/attachment.py | 6 +- .../code_interpreter_tool_definition.py | 22 + .../types/shared_params/completion_message.py | 6 +- .../function_call_tool_definition.py | 27 ++ .../shared_params/memory_tool_definition.py | 91 ++++ .../shared_params/photogen_tool_definition.py | 20 + .../shared_params/search_tool_definition.py | 24 + .../types/shared_params/system_message.py | 6 +- .../shared_params/tool_param_definition.py | 18 + .../shared_params/tool_response_message.py | 6 +- .../types/shared_params/user_message.py | 10 +- .../wolfram_alpha_tool_definition.py | 22 + .../types/shield_def_with_provider.py | 5 +- .../types/shield_def_with_provider_param.py | 6 +- .../types/shield_retrieve_params.py | 2 +- src/llama_stack_client/types/tool_response.py | 6 +- tests/api_resources/eval/test_jobs.py | 283 ++++++++++++ tests/api_resources/test_datasets.py | 8 + tests/api_resources/test_eval.py | 413 ++++++++++-------- tests/api_resources/test_models.py | 8 + tests/api_resources/test_safety.py | 16 +- tests/api_resources/test_scoring.py | 157 +++++-- tests/api_resources/test_scoring_functions.py | 140 +----- tests/api_resources/test_shields.py | 40 +- tests/test_client.py | 276 +++++------- tests/test_models.py | 21 +- tests/test_transform.py | 15 + 93 files changed, 2807 insertions(+), 1207 deletions(-) create mode 100644 src/llama_stack_client/resources/eval/jobs.py create mode 100644 src/llama_stack_client/types/eval/job_retrieve_params.py create mode 100644 src/llama_stack_client/types/eval/job_retrieve_response.py create mode 100644 src/llama_stack_client/types/eval/job_status_response.py create mode 100644 src/llama_stack_client/types/eval_evaluate_rows_params.py create mode 100644 src/llama_stack_client/types/eval_evaluate_rows_response.py create mode 100644 src/llama_stack_client/types/eval_run_eval_params.py create mode 100644 src/llama_stack_client/types/eval_run_eval_response.py create mode 100644 src/llama_stack_client/types/scoring_score_batch_response.py create mode 100644 src/llama_stack_client/types/scoring_score_response.py create mode 100644 src/llama_stack_client/types/shared/agent_config.py create mode 100644 src/llama_stack_client/types/shared/code_interpreter_tool_definition.py create mode 100644 src/llama_stack_client/types/shared/function_call_tool_definition.py create mode 100644 src/llama_stack_client/types/shared/memory_tool_definition.py create mode 100644 src/llama_stack_client/types/shared/photogen_tool_definition.py create mode 100644 src/llama_stack_client/types/shared/scoring_result.py create mode 100644 src/llama_stack_client/types/shared/search_tool_definition.py create mode 100644 src/llama_stack_client/types/shared/tool_param_definition.py create mode 100644 src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py create mode 100644 src/llama_stack_client/types/shared_params/agent_config.py create mode 100644 src/llama_stack_client/types/shared_params/code_interpreter_tool_definition.py create mode 100644 src/llama_stack_client/types/shared_params/function_call_tool_definition.py create mode 100644 src/llama_stack_client/types/shared_params/memory_tool_definition.py create mode 100644 src/llama_stack_client/types/shared_params/photogen_tool_definition.py create mode 100644 src/llama_stack_client/types/shared_params/search_tool_definition.py create mode 100644 src/llama_stack_client/types/shared_params/tool_param_definition.py create mode 100644 src/llama_stack_client/types/shared_params/wolfram_alpha_tool_definition.py create mode 100644 tests/api_resources/eval/test_jobs.py diff --git a/src/llama_stack_client/_client.py b/src/llama_stack_client/_client.py index 33e23819..41e29755 100644 --- a/src/llama_stack_client/_client.py +++ b/src/llama_stack_client/_client.py @@ -35,11 +35,12 @@ class LlamaStackClient(SyncAPIClient): agents: resources.AgentsResource batch_inferences: resources.BatchInferencesResource + datasets: resources.DatasetsResource + eval: resources.EvalResource inspect: resources.InspectResource inference: resources.InferenceResource memory: resources.MemoryResource memory_banks: resources.MemoryBanksResource - datasets: resources.DatasetsResource models: resources.ModelsResource post_training: resources.PostTrainingResource providers: resources.ProvidersResource @@ -51,7 +52,6 @@ class LlamaStackClient(SyncAPIClient): datasetio: resources.DatasetioResource scoring: resources.ScoringResource scoring_functions: resources.ScoringFunctionsResource - eval: resources.EvalResource with_raw_response: LlamaStackClientWithRawResponse with_streaming_response: LlamaStackClientWithStreamedResponse @@ -104,11 +104,12 @@ def __init__( self.agents = resources.AgentsResource(self) self.batch_inferences = resources.BatchInferencesResource(self) + self.datasets = resources.DatasetsResource(self) + self.eval = resources.EvalResource(self) self.inspect = resources.InspectResource(self) self.inference = resources.InferenceResource(self) self.memory = resources.MemoryResource(self) self.memory_banks = resources.MemoryBanksResource(self) - self.datasets = resources.DatasetsResource(self) self.models = resources.ModelsResource(self) self.post_training = resources.PostTrainingResource(self) self.providers = resources.ProvidersResource(self) @@ -120,7 +121,6 @@ def __init__( self.datasetio = resources.DatasetioResource(self) self.scoring = resources.ScoringResource(self) self.scoring_functions = resources.ScoringFunctionsResource(self) - self.eval = resources.EvalResource(self) self.with_raw_response = LlamaStackClientWithRawResponse(self) self.with_streaming_response = LlamaStackClientWithStreamedResponse(self) @@ -224,11 +224,12 @@ def _make_status_error( class AsyncLlamaStackClient(AsyncAPIClient): agents: resources.AsyncAgentsResource batch_inferences: resources.AsyncBatchInferencesResource + datasets: resources.AsyncDatasetsResource + eval: resources.AsyncEvalResource inspect: resources.AsyncInspectResource inference: resources.AsyncInferenceResource memory: resources.AsyncMemoryResource memory_banks: resources.AsyncMemoryBanksResource - datasets: resources.AsyncDatasetsResource models: resources.AsyncModelsResource post_training: resources.AsyncPostTrainingResource providers: resources.AsyncProvidersResource @@ -240,7 +241,6 @@ class AsyncLlamaStackClient(AsyncAPIClient): datasetio: resources.AsyncDatasetioResource scoring: resources.AsyncScoringResource scoring_functions: resources.AsyncScoringFunctionsResource - eval: resources.AsyncEvalResource with_raw_response: AsyncLlamaStackClientWithRawResponse with_streaming_response: AsyncLlamaStackClientWithStreamedResponse @@ -293,11 +293,12 @@ def __init__( self.agents = resources.AsyncAgentsResource(self) self.batch_inferences = resources.AsyncBatchInferencesResource(self) + self.datasets = resources.AsyncDatasetsResource(self) + self.eval = resources.AsyncEvalResource(self) self.inspect = resources.AsyncInspectResource(self) self.inference = resources.AsyncInferenceResource(self) self.memory = resources.AsyncMemoryResource(self) self.memory_banks = resources.AsyncMemoryBanksResource(self) - self.datasets = resources.AsyncDatasetsResource(self) self.models = resources.AsyncModelsResource(self) self.post_training = resources.AsyncPostTrainingResource(self) self.providers = resources.AsyncProvidersResource(self) @@ -309,7 +310,6 @@ def __init__( self.datasetio = resources.AsyncDatasetioResource(self) self.scoring = resources.AsyncScoringResource(self) self.scoring_functions = resources.AsyncScoringFunctionsResource(self) - self.eval = resources.AsyncEvalResource(self) self.with_raw_response = AsyncLlamaStackClientWithRawResponse(self) self.with_streaming_response = AsyncLlamaStackClientWithStreamedResponse(self) @@ -414,11 +414,12 @@ class LlamaStackClientWithRawResponse: def __init__(self, client: LlamaStackClient) -> None: self.agents = resources.AgentsResourceWithRawResponse(client.agents) self.batch_inferences = resources.BatchInferencesResourceWithRawResponse(client.batch_inferences) + self.datasets = resources.DatasetsResourceWithRawResponse(client.datasets) + self.eval = resources.EvalResourceWithRawResponse(client.eval) self.inspect = resources.InspectResourceWithRawResponse(client.inspect) self.inference = resources.InferenceResourceWithRawResponse(client.inference) self.memory = resources.MemoryResourceWithRawResponse(client.memory) self.memory_banks = resources.MemoryBanksResourceWithRawResponse(client.memory_banks) - self.datasets = resources.DatasetsResourceWithRawResponse(client.datasets) self.models = resources.ModelsResourceWithRawResponse(client.models) self.post_training = resources.PostTrainingResourceWithRawResponse(client.post_training) self.providers = resources.ProvidersResourceWithRawResponse(client.providers) @@ -432,18 +433,18 @@ def __init__(self, client: LlamaStackClient) -> None: self.datasetio = resources.DatasetioResourceWithRawResponse(client.datasetio) self.scoring = resources.ScoringResourceWithRawResponse(client.scoring) self.scoring_functions = resources.ScoringFunctionsResourceWithRawResponse(client.scoring_functions) - self.eval = resources.EvalResourceWithRawResponse(client.eval) class AsyncLlamaStackClientWithRawResponse: def __init__(self, client: AsyncLlamaStackClient) -> None: self.agents = resources.AsyncAgentsResourceWithRawResponse(client.agents) self.batch_inferences = resources.AsyncBatchInferencesResourceWithRawResponse(client.batch_inferences) + self.datasets = resources.AsyncDatasetsResourceWithRawResponse(client.datasets) + self.eval = resources.AsyncEvalResourceWithRawResponse(client.eval) self.inspect = resources.AsyncInspectResourceWithRawResponse(client.inspect) self.inference = resources.AsyncInferenceResourceWithRawResponse(client.inference) self.memory = resources.AsyncMemoryResourceWithRawResponse(client.memory) self.memory_banks = resources.AsyncMemoryBanksResourceWithRawResponse(client.memory_banks) - self.datasets = resources.AsyncDatasetsResourceWithRawResponse(client.datasets) self.models = resources.AsyncModelsResourceWithRawResponse(client.models) self.post_training = resources.AsyncPostTrainingResourceWithRawResponse(client.post_training) self.providers = resources.AsyncProvidersResourceWithRawResponse(client.providers) @@ -457,18 +458,18 @@ def __init__(self, client: AsyncLlamaStackClient) -> None: self.datasetio = resources.AsyncDatasetioResourceWithRawResponse(client.datasetio) self.scoring = resources.AsyncScoringResourceWithRawResponse(client.scoring) self.scoring_functions = resources.AsyncScoringFunctionsResourceWithRawResponse(client.scoring_functions) - self.eval = resources.AsyncEvalResourceWithRawResponse(client.eval) class LlamaStackClientWithStreamedResponse: def __init__(self, client: LlamaStackClient) -> None: self.agents = resources.AgentsResourceWithStreamingResponse(client.agents) self.batch_inferences = resources.BatchInferencesResourceWithStreamingResponse(client.batch_inferences) + self.datasets = resources.DatasetsResourceWithStreamingResponse(client.datasets) + self.eval = resources.EvalResourceWithStreamingResponse(client.eval) self.inspect = resources.InspectResourceWithStreamingResponse(client.inspect) self.inference = resources.InferenceResourceWithStreamingResponse(client.inference) self.memory = resources.MemoryResourceWithStreamingResponse(client.memory) self.memory_banks = resources.MemoryBanksResourceWithStreamingResponse(client.memory_banks) - self.datasets = resources.DatasetsResourceWithStreamingResponse(client.datasets) self.models = resources.ModelsResourceWithStreamingResponse(client.models) self.post_training = resources.PostTrainingResourceWithStreamingResponse(client.post_training) self.providers = resources.ProvidersResourceWithStreamingResponse(client.providers) @@ -482,18 +483,18 @@ def __init__(self, client: LlamaStackClient) -> None: self.datasetio = resources.DatasetioResourceWithStreamingResponse(client.datasetio) self.scoring = resources.ScoringResourceWithStreamingResponse(client.scoring) self.scoring_functions = resources.ScoringFunctionsResourceWithStreamingResponse(client.scoring_functions) - self.eval = resources.EvalResourceWithStreamingResponse(client.eval) class AsyncLlamaStackClientWithStreamedResponse: def __init__(self, client: AsyncLlamaStackClient) -> None: self.agents = resources.AsyncAgentsResourceWithStreamingResponse(client.agents) self.batch_inferences = resources.AsyncBatchInferencesResourceWithStreamingResponse(client.batch_inferences) + self.datasets = resources.AsyncDatasetsResourceWithStreamingResponse(client.datasets) + self.eval = resources.AsyncEvalResourceWithStreamingResponse(client.eval) self.inspect = resources.AsyncInspectResourceWithStreamingResponse(client.inspect) self.inference = resources.AsyncInferenceResourceWithStreamingResponse(client.inference) self.memory = resources.AsyncMemoryResourceWithStreamingResponse(client.memory) self.memory_banks = resources.AsyncMemoryBanksResourceWithStreamingResponse(client.memory_banks) - self.datasets = resources.AsyncDatasetsResourceWithStreamingResponse(client.datasets) self.models = resources.AsyncModelsResourceWithStreamingResponse(client.models) self.post_training = resources.AsyncPostTrainingResourceWithStreamingResponse(client.post_training) self.providers = resources.AsyncProvidersResourceWithStreamingResponse(client.providers) @@ -507,7 +508,6 @@ def __init__(self, client: AsyncLlamaStackClient) -> None: self.datasetio = resources.AsyncDatasetioResourceWithStreamingResponse(client.datasetio) self.scoring = resources.AsyncScoringResourceWithStreamingResponse(client.scoring) self.scoring_functions = resources.AsyncScoringFunctionsResourceWithStreamingResponse(client.scoring_functions) - self.eval = resources.AsyncEvalResourceWithStreamingResponse(client.eval) Client = LlamaStackClient diff --git a/src/llama_stack_client/_compat.py b/src/llama_stack_client/_compat.py index d89920d9..4794129c 100644 --- a/src/llama_stack_client/_compat.py +++ b/src/llama_stack_client/_compat.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Any, Union, Generic, TypeVar, Callable, cast, overload from datetime import date, datetime -from typing_extensions import Self +from typing_extensions import Self, Literal import pydantic from pydantic.fields import FieldInfo @@ -137,9 +137,11 @@ def model_dump( exclude_unset: bool = False, exclude_defaults: bool = False, warnings: bool = True, + mode: Literal["json", "python"] = "python", ) -> dict[str, Any]: - if PYDANTIC_V2: + if PYDANTIC_V2 or hasattr(model, "model_dump"): return model.model_dump( + mode=mode, exclude=exclude, exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, diff --git a/src/llama_stack_client/_models.py b/src/llama_stack_client/_models.py index 42551b76..6cb469e2 100644 --- a/src/llama_stack_client/_models.py +++ b/src/llama_stack_client/_models.py @@ -37,6 +37,7 @@ PropertyInfo, is_list, is_given, + json_safe, lru_cache, is_mapping, parse_date, @@ -279,8 +280,8 @@ def model_dump( Returns: A dictionary representation of the model. """ - if mode != "python": - raise ValueError("mode is only supported in Pydantic v2") + if mode not in {"json", "python"}: + raise ValueError("mode must be either 'json' or 'python'") if round_trip != False: raise ValueError("round_trip is only supported in Pydantic v2") if warnings != True: @@ -289,7 +290,7 @@ def model_dump( raise ValueError("context is only supported in Pydantic v2") if serialize_as_any != False: raise ValueError("serialize_as_any is only supported in Pydantic v2") - return super().dict( # pyright: ignore[reportDeprecated] + dumped = super().dict( # pyright: ignore[reportDeprecated] include=include, exclude=exclude, by_alias=by_alias, @@ -298,6 +299,8 @@ def model_dump( exclude_none=exclude_none, ) + return cast(dict[str, Any], json_safe(dumped)) if mode == "json" else dumped + @override def model_dump_json( self, diff --git a/src/llama_stack_client/_utils/__init__.py b/src/llama_stack_client/_utils/__init__.py index 3efe66c8..a7cff3c0 100644 --- a/src/llama_stack_client/_utils/__init__.py +++ b/src/llama_stack_client/_utils/__init__.py @@ -6,6 +6,7 @@ is_list as is_list, is_given as is_given, is_tuple as is_tuple, + json_safe as json_safe, lru_cache as lru_cache, is_mapping as is_mapping, is_tuple_t as is_tuple_t, diff --git a/src/llama_stack_client/_utils/_transform.py b/src/llama_stack_client/_utils/_transform.py index 47e262a5..d7c05345 100644 --- a/src/llama_stack_client/_utils/_transform.py +++ b/src/llama_stack_client/_utils/_transform.py @@ -173,6 +173,11 @@ def _transform_recursive( # Iterable[T] or (is_iterable_type(stripped_type) and is_iterable(data) and not isinstance(data, str)) ): + # dicts are technically iterable, but it is an iterable on the keys of the dict and is not usually + # intended as an iterable, so we don't transform it. + if isinstance(data, dict): + return cast(object, data) + inner_type = extract_type_arg(stripped_type, 0) return [_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data] @@ -186,7 +191,7 @@ def _transform_recursive( return data if isinstance(data, pydantic.BaseModel): - return model_dump(data, exclude_unset=True) + return model_dump(data, exclude_unset=True, mode="json") annotated_type = _get_annotated_type(annotation) if annotated_type is None: @@ -324,7 +329,7 @@ async def _async_transform_recursive( return data if isinstance(data, pydantic.BaseModel): - return model_dump(data, exclude_unset=True) + return model_dump(data, exclude_unset=True, mode="json") annotated_type = _get_annotated_type(annotation) if annotated_type is None: diff --git a/src/llama_stack_client/_utils/_utils.py b/src/llama_stack_client/_utils/_utils.py index 0bba17ca..e5811bba 100644 --- a/src/llama_stack_client/_utils/_utils.py +++ b/src/llama_stack_client/_utils/_utils.py @@ -16,6 +16,7 @@ overload, ) from pathlib import Path +from datetime import date, datetime from typing_extensions import TypeGuard import sniffio @@ -395,3 +396,19 @@ def lru_cache(*, maxsize: int | None = 128) -> Callable[[CallableT], CallableT]: maxsize=maxsize, ) return cast(Any, wrapper) # type: ignore[no-any-return] + + +def json_safe(data: object) -> object: + """Translates a mapping / sequence recursively in the same fashion + as `pydantic` v2's `model_dump(mode="json")`. + """ + if is_mapping(data): + return {json_safe(key): json_safe(value) for key, value in data.items()} + + if is_iterable(data) and not isinstance(data, (str, bytes, bytearray)): + return [json_safe(item) for item in data] + + if isinstance(data, (datetime, date)): + return data.isoformat() + + return data diff --git a/src/llama_stack_client/resources/__init__.py b/src/llama_stack_client/resources/__init__.py index f5cdbbeb..655c3975 100644 --- a/src/llama_stack_client/resources/__init__.py +++ b/src/llama_stack_client/resources/__init__.py @@ -166,6 +166,18 @@ "AsyncBatchInferencesResourceWithRawResponse", "BatchInferencesResourceWithStreamingResponse", "AsyncBatchInferencesResourceWithStreamingResponse", + "DatasetsResource", + "AsyncDatasetsResource", + "DatasetsResourceWithRawResponse", + "AsyncDatasetsResourceWithRawResponse", + "DatasetsResourceWithStreamingResponse", + "AsyncDatasetsResourceWithStreamingResponse", + "EvalResource", + "AsyncEvalResource", + "EvalResourceWithRawResponse", + "AsyncEvalResourceWithRawResponse", + "EvalResourceWithStreamingResponse", + "AsyncEvalResourceWithStreamingResponse", "InspectResource", "AsyncInspectResource", "InspectResourceWithRawResponse", @@ -190,12 +202,6 @@ "AsyncMemoryBanksResourceWithRawResponse", "MemoryBanksResourceWithStreamingResponse", "AsyncMemoryBanksResourceWithStreamingResponse", - "DatasetsResource", - "AsyncDatasetsResource", - "DatasetsResourceWithRawResponse", - "AsyncDatasetsResourceWithRawResponse", - "DatasetsResourceWithStreamingResponse", - "AsyncDatasetsResourceWithStreamingResponse", "ModelsResource", "AsyncModelsResource", "ModelsResourceWithRawResponse", @@ -262,10 +268,4 @@ "AsyncScoringFunctionsResourceWithRawResponse", "ScoringFunctionsResourceWithStreamingResponse", "AsyncScoringFunctionsResourceWithStreamingResponse", - "EvalResource", - "AsyncEvalResource", - "EvalResourceWithRawResponse", - "AsyncEvalResourceWithRawResponse", - "EvalResourceWithStreamingResponse", - "AsyncEvalResourceWithStreamingResponse", ] diff --git a/src/llama_stack_client/resources/agents/agents.py b/src/llama_stack_client/resources/agents/agents.py index 2d57f822..9945a92a 100644 --- a/src/llama_stack_client/resources/agents/agents.py +++ b/src/llama_stack_client/resources/agents/agents.py @@ -45,6 +45,7 @@ ) from ..._base_client import make_request_options from ...types.agent_create_response import AgentCreateResponse +from ...types.shared_params.agent_config import AgentConfig __all__ = ["AgentsResource", "AsyncAgentsResource"] @@ -84,7 +85,7 @@ def with_streaming_response(self) -> AgentsResourceWithStreamingResponse: def create( self, *, - agent_config: agent_create_params.AgentConfig, + agent_config: AgentConfig, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -188,7 +189,7 @@ def with_streaming_response(self) -> AsyncAgentsResourceWithStreamingResponse: async def create( self, *, - agent_config: agent_create_params.AgentConfig, + agent_config: AgentConfig, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. diff --git a/src/llama_stack_client/resources/agents/turn.py b/src/llama_stack_client/resources/agents/turn.py index f63d356b..4738b0ff 100644 --- a/src/llama_stack_client/resources/agents/turn.py +++ b/src/llama_stack_client/resources/agents/turn.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Iterable +from typing import Any, Iterable, cast from typing_extensions import Literal, overload import httpx @@ -161,24 +161,29 @@ def create( **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), **(extra_headers or {}), } - return self._post( - "/agents/turn/create", - body=maybe_transform( - { - "agent_id": agent_id, - "messages": messages, - "session_id": session_id, - "attachments": attachments, - "stream": stream, - }, - turn_create_params.TurnCreateParams, - ), - options=make_request_options( - extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + return cast( + TurnCreateResponse, + self._post( + "/agents/turn/create", + body=maybe_transform( + { + "agent_id": agent_id, + "messages": messages, + "session_id": session_id, + "attachments": attachments, + "stream": stream, + }, + turn_create_params.TurnCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=cast( + Any, TurnCreateResponse + ), # Union types cannot be passed in as arguments in the type system + stream=stream or False, + stream_cls=Stream[TurnCreateResponse], ), - cast_to=TurnCreateResponse, - stream=stream or False, - stream_cls=Stream[TurnCreateResponse], ) def retrieve( @@ -358,24 +363,29 @@ async def create( **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), **(extra_headers or {}), } - return await self._post( - "/agents/turn/create", - body=await async_maybe_transform( - { - "agent_id": agent_id, - "messages": messages, - "session_id": session_id, - "attachments": attachments, - "stream": stream, - }, - turn_create_params.TurnCreateParams, - ), - options=make_request_options( - extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + return cast( + TurnCreateResponse, + await self._post( + "/agents/turn/create", + body=await async_maybe_transform( + { + "agent_id": agent_id, + "messages": messages, + "session_id": session_id, + "attachments": attachments, + "stream": stream, + }, + turn_create_params.TurnCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=cast( + Any, TurnCreateResponse + ), # Union types cannot be passed in as arguments in the type system + stream=stream or False, + stream_cls=AsyncStream[TurnCreateResponse], ), - cast_to=TurnCreateResponse, - stream=stream or False, - stream_cls=AsyncStream[TurnCreateResponse], ) async def retrieve( diff --git a/src/llama_stack_client/resources/eval/__init__.py b/src/llama_stack_client/resources/eval/__init__.py index 771fdbb4..f6473395 100644 --- a/src/llama_stack_client/resources/eval/__init__.py +++ b/src/llama_stack_client/resources/eval/__init__.py @@ -1,13 +1,5 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from .job import ( - JobResource, - AsyncJobResource, - JobResourceWithRawResponse, - AsyncJobResourceWithRawResponse, - JobResourceWithStreamingResponse, - AsyncJobResourceWithStreamingResponse, -) from .eval import ( EvalResource, AsyncEvalResource, @@ -16,14 +8,22 @@ EvalResourceWithStreamingResponse, AsyncEvalResourceWithStreamingResponse, ) +from .jobs import ( + JobsResource, + AsyncJobsResource, + JobsResourceWithRawResponse, + AsyncJobsResourceWithRawResponse, + JobsResourceWithStreamingResponse, + AsyncJobsResourceWithStreamingResponse, +) __all__ = [ - "JobResource", - "AsyncJobResource", - "JobResourceWithRawResponse", - "AsyncJobResourceWithRawResponse", - "JobResourceWithStreamingResponse", - "AsyncJobResourceWithStreamingResponse", + "JobsResource", + "AsyncJobsResource", + "JobsResourceWithRawResponse", + "AsyncJobsResourceWithRawResponse", + "JobsResourceWithStreamingResponse", + "AsyncJobsResourceWithStreamingResponse", "EvalResource", "AsyncEvalResource", "EvalResourceWithRawResponse", diff --git a/src/llama_stack_client/resources/eval/eval.py b/src/llama_stack_client/resources/eval/eval.py index 1e4ff7eb..77cb1cb6 100644 --- a/src/llama_stack_client/resources/eval/eval.py +++ b/src/llama_stack_client/resources/eval/eval.py @@ -6,15 +6,15 @@ import httpx -from .job import ( - JobResource, - AsyncJobResource, - JobResourceWithRawResponse, - AsyncJobResourceWithRawResponse, - JobResourceWithStreamingResponse, - AsyncJobResourceWithStreamingResponse, +from .jobs import ( + JobsResource, + AsyncJobsResource, + JobsResourceWithRawResponse, + AsyncJobsResourceWithRawResponse, + JobsResourceWithStreamingResponse, + AsyncJobsResourceWithStreamingResponse, ) -from ...types import eval_evaluate_params, eval_evaluate_batch_params +from ...types import eval_run_eval_params, eval_evaluate_rows_params from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven from ..._utils import ( maybe_transform, @@ -29,17 +29,17 @@ async_to_raw_response_wrapper, async_to_streamed_response_wrapper, ) -from ...types.job import Job from ..._base_client import make_request_options -from ...types.eval_evaluate_response import EvalEvaluateResponse +from ...types.eval_run_eval_response import EvalRunEvalResponse +from ...types.eval_evaluate_rows_response import EvalEvaluateRowsResponse __all__ = ["EvalResource", "AsyncEvalResource"] class EvalResource(SyncAPIResource): @cached_property - def job(self) -> JobResource: - return JobResource(self._client) + def jobs(self) -> JobsResource: + return JobsResource(self._client) @cached_property def with_raw_response(self) -> EvalResourceWithRawResponse: @@ -60,12 +60,13 @@ def with_streaming_response(self) -> EvalResourceWithStreamingResponse: """ return EvalResourceWithStreamingResponse(self) - def evaluate( + def evaluate_rows( self, *, - candidate: eval_evaluate_params.Candidate, input_rows: Iterable[Dict[str, Union[bool, float, str, Iterable[object], object, None]]], scoring_functions: List[str], + task_config: eval_evaluate_rows_params.TaskConfig, + task_id: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -73,7 +74,7 @@ def evaluate( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> EvalEvaluateResponse: + ) -> EvalEvaluateRowsResponse: """ Args: extra_headers: Send extra headers @@ -89,27 +90,27 @@ def evaluate( **(extra_headers or {}), } return self._post( - "/eval/evaluate", + "/eval/evaluate_rows", body=maybe_transform( { - "candidate": candidate, "input_rows": input_rows, "scoring_functions": scoring_functions, + "task_config": task_config, + "task_id": task_id, }, - eval_evaluate_params.EvalEvaluateParams, + eval_evaluate_rows_params.EvalEvaluateRowsParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=EvalEvaluateResponse, + cast_to=EvalEvaluateRowsResponse, ) - def evaluate_batch( + def run_eval( self, *, - candidate: eval_evaluate_batch_params.Candidate, - dataset_id: str, - scoring_functions: List[str], + task_config: eval_run_eval_params.TaskConfig, + task_id: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -117,7 +118,7 @@ def evaluate_batch( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> Job: + ) -> EvalRunEvalResponse: """ Args: extra_headers: Send extra headers @@ -133,26 +134,25 @@ def evaluate_batch( **(extra_headers or {}), } return self._post( - "/eval/evaluate_batch", + "/eval/run_eval", body=maybe_transform( { - "candidate": candidate, - "dataset_id": dataset_id, - "scoring_functions": scoring_functions, + "task_config": task_config, + "task_id": task_id, }, - eval_evaluate_batch_params.EvalEvaluateBatchParams, + eval_run_eval_params.EvalRunEvalParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=Job, + cast_to=EvalRunEvalResponse, ) class AsyncEvalResource(AsyncAPIResource): @cached_property - def job(self) -> AsyncJobResource: - return AsyncJobResource(self._client) + def jobs(self) -> AsyncJobsResource: + return AsyncJobsResource(self._client) @cached_property def with_raw_response(self) -> AsyncEvalResourceWithRawResponse: @@ -173,12 +173,13 @@ def with_streaming_response(self) -> AsyncEvalResourceWithStreamingResponse: """ return AsyncEvalResourceWithStreamingResponse(self) - async def evaluate( + async def evaluate_rows( self, *, - candidate: eval_evaluate_params.Candidate, input_rows: Iterable[Dict[str, Union[bool, float, str, Iterable[object], object, None]]], scoring_functions: List[str], + task_config: eval_evaluate_rows_params.TaskConfig, + task_id: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -186,7 +187,7 @@ async def evaluate( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> EvalEvaluateResponse: + ) -> EvalEvaluateRowsResponse: """ Args: extra_headers: Send extra headers @@ -202,27 +203,27 @@ async def evaluate( **(extra_headers or {}), } return await self._post( - "/eval/evaluate", + "/eval/evaluate_rows", body=await async_maybe_transform( { - "candidate": candidate, "input_rows": input_rows, "scoring_functions": scoring_functions, + "task_config": task_config, + "task_id": task_id, }, - eval_evaluate_params.EvalEvaluateParams, + eval_evaluate_rows_params.EvalEvaluateRowsParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=EvalEvaluateResponse, + cast_to=EvalEvaluateRowsResponse, ) - async def evaluate_batch( + async def run_eval( self, *, - candidate: eval_evaluate_batch_params.Candidate, - dataset_id: str, - scoring_functions: List[str], + task_config: eval_run_eval_params.TaskConfig, + task_id: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -230,7 +231,7 @@ async def evaluate_batch( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> Job: + ) -> EvalRunEvalResponse: """ Args: extra_headers: Send extra headers @@ -246,19 +247,18 @@ async def evaluate_batch( **(extra_headers or {}), } return await self._post( - "/eval/evaluate_batch", + "/eval/run_eval", body=await async_maybe_transform( { - "candidate": candidate, - "dataset_id": dataset_id, - "scoring_functions": scoring_functions, + "task_config": task_config, + "task_id": task_id, }, - eval_evaluate_batch_params.EvalEvaluateBatchParams, + eval_run_eval_params.EvalRunEvalParams, ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=Job, + cast_to=EvalRunEvalResponse, ) @@ -266,61 +266,61 @@ class EvalResourceWithRawResponse: def __init__(self, eval: EvalResource) -> None: self._eval = eval - self.evaluate = to_raw_response_wrapper( - eval.evaluate, + self.evaluate_rows = to_raw_response_wrapper( + eval.evaluate_rows, ) - self.evaluate_batch = to_raw_response_wrapper( - eval.evaluate_batch, + self.run_eval = to_raw_response_wrapper( + eval.run_eval, ) @cached_property - def job(self) -> JobResourceWithRawResponse: - return JobResourceWithRawResponse(self._eval.job) + def jobs(self) -> JobsResourceWithRawResponse: + return JobsResourceWithRawResponse(self._eval.jobs) class AsyncEvalResourceWithRawResponse: def __init__(self, eval: AsyncEvalResource) -> None: self._eval = eval - self.evaluate = async_to_raw_response_wrapper( - eval.evaluate, + self.evaluate_rows = async_to_raw_response_wrapper( + eval.evaluate_rows, ) - self.evaluate_batch = async_to_raw_response_wrapper( - eval.evaluate_batch, + self.run_eval = async_to_raw_response_wrapper( + eval.run_eval, ) @cached_property - def job(self) -> AsyncJobResourceWithRawResponse: - return AsyncJobResourceWithRawResponse(self._eval.job) + def jobs(self) -> AsyncJobsResourceWithRawResponse: + return AsyncJobsResourceWithRawResponse(self._eval.jobs) class EvalResourceWithStreamingResponse: def __init__(self, eval: EvalResource) -> None: self._eval = eval - self.evaluate = to_streamed_response_wrapper( - eval.evaluate, + self.evaluate_rows = to_streamed_response_wrapper( + eval.evaluate_rows, ) - self.evaluate_batch = to_streamed_response_wrapper( - eval.evaluate_batch, + self.run_eval = to_streamed_response_wrapper( + eval.run_eval, ) @cached_property - def job(self) -> JobResourceWithStreamingResponse: - return JobResourceWithStreamingResponse(self._eval.job) + def jobs(self) -> JobsResourceWithStreamingResponse: + return JobsResourceWithStreamingResponse(self._eval.jobs) class AsyncEvalResourceWithStreamingResponse: def __init__(self, eval: AsyncEvalResource) -> None: self._eval = eval - self.evaluate = async_to_streamed_response_wrapper( - eval.evaluate, + self.evaluate_rows = async_to_streamed_response_wrapper( + eval.evaluate_rows, ) - self.evaluate_batch = async_to_streamed_response_wrapper( - eval.evaluate_batch, + self.run_eval = async_to_streamed_response_wrapper( + eval.run_eval, ) @cached_property - def job(self) -> AsyncJobResourceWithStreamingResponse: - return AsyncJobResourceWithStreamingResponse(self._eval.job) + def jobs(self) -> AsyncJobsResourceWithStreamingResponse: + return AsyncJobsResourceWithStreamingResponse(self._eval.jobs) diff --git a/src/llama_stack_client/resources/eval/jobs.py b/src/llama_stack_client/resources/eval/jobs.py new file mode 100644 index 00000000..b8604cb8 --- /dev/null +++ b/src/llama_stack_client/resources/eval/jobs.py @@ -0,0 +1,396 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Optional + +import httpx + +from ..._types import NOT_GIVEN, Body, Query, Headers, NoneType, NotGiven +from ..._utils import ( + maybe_transform, + strip_not_given, + async_maybe_transform, +) +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from ...types.eval import job_cancel_params, job_status_params, job_retrieve_params +from ..._base_client import make_request_options +from ...types.eval.job_status_response import JobStatusResponse +from ...types.eval.job_retrieve_response import JobRetrieveResponse + +__all__ = ["JobsResource", "AsyncJobsResource"] + + +class JobsResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> JobsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#accessing-raw-response-data-eg-headers + """ + return JobsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> JobsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#with_streaming_response + """ + return JobsResourceWithStreamingResponse(self) + + def retrieve( + self, + *, + job_id: str, + task_id: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> JobRetrieveResponse: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return self._get( + "/eval/job/result", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "job_id": job_id, + "task_id": task_id, + }, + job_retrieve_params.JobRetrieveParams, + ), + ), + cast_to=JobRetrieveResponse, + ) + + def cancel( + self, + *, + job_id: str, + task_id: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> None: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return self._post( + "/eval/job/cancel", + body=maybe_transform( + { + "job_id": job_id, + "task_id": task_id, + }, + job_cancel_params.JobCancelParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + def status( + self, + *, + job_id: str, + task_id: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Optional[JobStatusResponse]: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return self._get( + "/eval/job/status", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "job_id": job_id, + "task_id": task_id, + }, + job_status_params.JobStatusParams, + ), + ), + cast_to=JobStatusResponse, + ) + + +class AsyncJobsResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncJobsResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#accessing-raw-response-data-eg-headers + """ + return AsyncJobsResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncJobsResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#with_streaming_response + """ + return AsyncJobsResourceWithStreamingResponse(self) + + async def retrieve( + self, + *, + job_id: str, + task_id: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> JobRetrieveResponse: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return await self._get( + "/eval/job/result", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform( + { + "job_id": job_id, + "task_id": task_id, + }, + job_retrieve_params.JobRetrieveParams, + ), + ), + cast_to=JobRetrieveResponse, + ) + + async def cancel( + self, + *, + job_id: str, + task_id: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> None: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return await self._post( + "/eval/job/cancel", + body=await async_maybe_transform( + { + "job_id": job_id, + "task_id": task_id, + }, + job_cancel_params.JobCancelParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + async def status( + self, + *, + job_id: str, + task_id: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Optional[JobStatusResponse]: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return await self._get( + "/eval/job/status", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform( + { + "job_id": job_id, + "task_id": task_id, + }, + job_status_params.JobStatusParams, + ), + ), + cast_to=JobStatusResponse, + ) + + +class JobsResourceWithRawResponse: + def __init__(self, jobs: JobsResource) -> None: + self._jobs = jobs + + self.retrieve = to_raw_response_wrapper( + jobs.retrieve, + ) + self.cancel = to_raw_response_wrapper( + jobs.cancel, + ) + self.status = to_raw_response_wrapper( + jobs.status, + ) + + +class AsyncJobsResourceWithRawResponse: + def __init__(self, jobs: AsyncJobsResource) -> None: + self._jobs = jobs + + self.retrieve = async_to_raw_response_wrapper( + jobs.retrieve, + ) + self.cancel = async_to_raw_response_wrapper( + jobs.cancel, + ) + self.status = async_to_raw_response_wrapper( + jobs.status, + ) + + +class JobsResourceWithStreamingResponse: + def __init__(self, jobs: JobsResource) -> None: + self._jobs = jobs + + self.retrieve = to_streamed_response_wrapper( + jobs.retrieve, + ) + self.cancel = to_streamed_response_wrapper( + jobs.cancel, + ) + self.status = to_streamed_response_wrapper( + jobs.status, + ) + + +class AsyncJobsResourceWithStreamingResponse: + def __init__(self, jobs: AsyncJobsResource) -> None: + self._jobs = jobs + + self.retrieve = async_to_streamed_response_wrapper( + jobs.retrieve, + ) + self.cancel = async_to_streamed_response_wrapper( + jobs.cancel, + ) + self.status = async_to_streamed_response_wrapper( + jobs.status, + ) diff --git a/src/llama_stack_client/resources/safety.py b/src/llama_stack_client/resources/safety.py index 3f964c58..8532c25e 100644 --- a/src/llama_stack_client/resources/safety.py +++ b/src/llama_stack_client/resources/safety.py @@ -50,9 +50,9 @@ def with_streaming_response(self) -> SafetyResourceWithStreamingResponse: def run_shield( self, *, + identifier: str, messages: Iterable[safety_run_shield_params.Message], params: Dict[str, Union[bool, float, str, Iterable[object], object, None]], - shield_type: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -79,9 +79,9 @@ def run_shield( "/safety/run_shield", body=maybe_transform( { + "identifier": identifier, "messages": messages, "params": params, - "shield_type": shield_type, }, safety_run_shield_params.SafetyRunShieldParams, ), @@ -115,9 +115,9 @@ def with_streaming_response(self) -> AsyncSafetyResourceWithStreamingResponse: async def run_shield( self, *, + identifier: str, messages: Iterable[safety_run_shield_params.Message], params: Dict[str, Union[bool, float, str, Iterable[object], object, None]], - shield_type: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -144,9 +144,9 @@ async def run_shield( "/safety/run_shield", body=await async_maybe_transform( { + "identifier": identifier, "messages": messages, "params": params, - "shield_type": shield_type, }, safety_run_shield_params.SafetyRunShieldParams, ), diff --git a/src/llama_stack_client/resources/scoring.py b/src/llama_stack_client/resources/scoring.py index ce77012f..6d39faa1 100644 --- a/src/llama_stack_client/resources/scoring.py +++ b/src/llama_stack_client/resources/scoring.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Dict, List, Union, Iterable +from typing import Dict, Union, Iterable, Optional import httpx @@ -22,8 +22,8 @@ async_to_streamed_response_wrapper, ) from .._base_client import make_request_options -from ..types.score_response import ScoreResponse -from ..types.score_batch_response import ScoreBatchResponse +from ..types.scoring_score_response import ScoringScoreResponse +from ..types.scoring_score_batch_response import ScoringScoreBatchResponse __all__ = ["ScoringResource", "AsyncScoringResource"] @@ -52,7 +52,7 @@ def score( self, *, input_rows: Iterable[Dict[str, Union[bool, float, str, Iterable[object], object, None]]], - scoring_functions: List[str], + scoring_functions: Dict[str, Optional[scoring_score_params.ScoringFunctions]], x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -60,7 +60,7 @@ def score( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> ScoreResponse: + ) -> ScoringScoreResponse: """ Args: extra_headers: Send extra headers @@ -87,7 +87,7 @@ def score( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ScoreResponse, + cast_to=ScoringScoreResponse, ) def score_batch( @@ -95,7 +95,7 @@ def score_batch( *, dataset_id: str, save_results_dataset: bool, - scoring_functions: List[str], + scoring_functions: Dict[str, Optional[scoring_score_batch_params.ScoringFunctions]], x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -103,7 +103,7 @@ def score_batch( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> ScoreBatchResponse: + ) -> ScoringScoreBatchResponse: """ Args: extra_headers: Send extra headers @@ -131,7 +131,7 @@ def score_batch( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ScoreBatchResponse, + cast_to=ScoringScoreBatchResponse, ) @@ -159,7 +159,7 @@ async def score( self, *, input_rows: Iterable[Dict[str, Union[bool, float, str, Iterable[object], object, None]]], - scoring_functions: List[str], + scoring_functions: Dict[str, Optional[scoring_score_params.ScoringFunctions]], x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -167,7 +167,7 @@ async def score( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> ScoreResponse: + ) -> ScoringScoreResponse: """ Args: extra_headers: Send extra headers @@ -194,7 +194,7 @@ async def score( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ScoreResponse, + cast_to=ScoringScoreResponse, ) async def score_batch( @@ -202,7 +202,7 @@ async def score_batch( *, dataset_id: str, save_results_dataset: bool, - scoring_functions: List[str], + scoring_functions: Dict[str, Optional[scoring_score_batch_params.ScoringFunctions]], x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -210,7 +210,7 @@ async def score_batch( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, - ) -> ScoreBatchResponse: + ) -> ScoringScoreBatchResponse: """ Args: extra_headers: Send extra headers @@ -238,7 +238,7 @@ async def score_batch( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=ScoreBatchResponse, + cast_to=ScoringScoreBatchResponse, ) diff --git a/src/llama_stack_client/resources/shields.py b/src/llama_stack_client/resources/shields.py index 40eb8108..cb285114 100644 --- a/src/llama_stack_client/resources/shields.py +++ b/src/llama_stack_client/resources/shields.py @@ -51,7 +51,7 @@ def with_streaming_response(self) -> ShieldsResourceWithStreamingResponse: def retrieve( self, *, - shield_type: str, + identifier: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -81,7 +81,7 @@ def retrieve( extra_query=extra_query, extra_body=extra_body, timeout=timeout, - query=maybe_transform({"shield_type": shield_type}, shield_retrieve_params.ShieldRetrieveParams), + query=maybe_transform({"identifier": identifier}, shield_retrieve_params.ShieldRetrieveParams), ), cast_to=ShieldDefWithProvider, ) @@ -180,7 +180,7 @@ def with_streaming_response(self) -> AsyncShieldsResourceWithStreamingResponse: async def retrieve( self, *, - shield_type: str, + identifier: str, x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -211,7 +211,7 @@ async def retrieve( extra_body=extra_body, timeout=timeout, query=await async_maybe_transform( - {"shield_type": shield_type}, shield_retrieve_params.ShieldRetrieveParams + {"identifier": identifier}, shield_retrieve_params.ShieldRetrieveParams ), ), cast_to=ShieldDefWithProvider, diff --git a/src/llama_stack_client/types/__init__.py b/src/llama_stack_client/types/__init__.py index e5a02c3e..4c72194a 100644 --- a/src/llama_stack_client/types/__init__.py +++ b/src/llama_stack_client/types/__init__.py @@ -2,30 +2,37 @@ from __future__ import annotations -from .job import Job as Job from .trace import Trace as Trace from .shared import ( ToolCall as ToolCall, Attachment as Attachment, ImageMedia as ImageMedia, + AgentConfig as AgentConfig, UserMessage as UserMessage, + ScoringResult as ScoringResult, SystemMessage as SystemMessage, SamplingParams as SamplingParams, BatchCompletion as BatchCompletion, SafetyViolation as SafetyViolation, CompletionMessage as CompletionMessage, GraphMemoryBankDef as GraphMemoryBankDef, + ToolParamDefinition as ToolParamDefinition, ToolResponseMessage as ToolResponseMessage, VectorMemoryBankDef as VectorMemoryBankDef, KeywordMemoryBankDef as KeywordMemoryBankDef, + MemoryToolDefinition as MemoryToolDefinition, + SearchToolDefinition as SearchToolDefinition, KeyValueMemoryBankDef as KeyValueMemoryBankDef, + PhotogenToolDefinition as PhotogenToolDefinition, + FunctionCallToolDefinition as FunctionCallToolDefinition, + WolframAlphaToolDefinition as WolframAlphaToolDefinition, + CodeInterpreterToolDefinition as CodeInterpreterToolDefinition, ) from .route_info import RouteInfo as RouteInfo from .health_info import HealthInfo as HealthInfo from .provider_info import ProviderInfo as ProviderInfo from .tool_response import ToolResponse as ToolResponse from .inference_step import InferenceStep as InferenceStep -from .score_response import ScoreResponse as ScoreResponse from .token_log_probs import TokenLogProbs as TokenLogProbs from .shield_call_step import ShieldCallStep as ShieldCallStep from .post_training_job import PostTrainingJob as PostTrainingJob @@ -37,9 +44,8 @@ from .route_list_response import RouteListResponse as RouteListResponse from .run_shield_response import RunShieldResponse as RunShieldResponse from .tool_execution_step import ToolExecutionStep as ToolExecutionStep -from .eval_evaluate_params import EvalEvaluateParams as EvalEvaluateParams +from .eval_run_eval_params import EvalRunEvalParams as EvalRunEvalParams from .memory_insert_params import MemoryInsertParams as MemoryInsertParams -from .score_batch_response import ScoreBatchResponse as ScoreBatchResponse from .scoring_score_params import ScoringScoreParams as ScoringScoreParams from .agent_create_response import AgentCreateResponse as AgentCreateResponse from .dataset_list_response import DatasetListResponse as DatasetListResponse @@ -47,8 +53,9 @@ from .model_register_params import ModelRegisterParams as ModelRegisterParams from .model_retrieve_params import ModelRetrieveParams as ModelRetrieveParams from .paginated_rows_result import PaginatedRowsResult as PaginatedRowsResult -from .eval_evaluate_response import EvalEvaluateResponse as EvalEvaluateResponse +from .eval_run_eval_response import EvalRunEvalResponse as EvalRunEvalResponse from .provider_list_response import ProviderListResponse as ProviderListResponse +from .scoring_score_response import ScoringScoreResponse as ScoringScoreResponse from .shield_register_params import ShieldRegisterParams as ShieldRegisterParams from .shield_retrieve_params import ShieldRetrieveParams as ShieldRetrieveParams from .dataset_register_params import DatasetRegisterParams as DatasetRegisterParams @@ -58,16 +65,18 @@ from .safety_run_shield_params import SafetyRunShieldParams as SafetyRunShieldParams from .shield_def_with_provider import ShieldDefWithProvider as ShieldDefWithProvider from .dataset_retrieve_response import DatasetRetrieveResponse as DatasetRetrieveResponse +from .eval_evaluate_rows_params import EvalEvaluateRowsParams as EvalEvaluateRowsParams from .memory_bank_list_response import MemoryBankListResponse as MemoryBankListResponse -from .eval_evaluate_batch_params import EvalEvaluateBatchParams as EvalEvaluateBatchParams from .scoring_score_batch_params import ScoringScoreBatchParams as ScoringScoreBatchParams from .telemetry_get_trace_params import TelemetryGetTraceParams as TelemetryGetTraceParams from .telemetry_log_event_params import TelemetryLogEventParams as TelemetryLogEventParams +from .eval_evaluate_rows_response import EvalEvaluateRowsResponse as EvalEvaluateRowsResponse from .inference_completion_params import InferenceCompletionParams as InferenceCompletionParams from .inference_embeddings_params import InferenceEmbeddingsParams as InferenceEmbeddingsParams from .memory_bank_register_params import MemoryBankRegisterParams as MemoryBankRegisterParams from .memory_bank_retrieve_params import MemoryBankRetrieveParams as MemoryBankRetrieveParams from .scoring_fn_def_with_provider import ScoringFnDefWithProvider as ScoringFnDefWithProvider +from .scoring_score_batch_response import ScoringScoreBatchResponse as ScoringScoreBatchResponse from .inference_completion_response import InferenceCompletionResponse as InferenceCompletionResponse from .memory_bank_retrieve_response import MemoryBankRetrieveResponse as MemoryBankRetrieveResponse from .model_def_with_provider_param import ModelDefWithProviderParam as ModelDefWithProviderParam diff --git a/src/llama_stack_client/types/agent_create_params.py b/src/llama_stack_client/types/agent_create_params.py index bba57077..fdbc8fe3 100644 --- a/src/llama_stack_client/types/agent_create_params.py +++ b/src/llama_stack_client/types/agent_create_params.py @@ -2,231 +2,15 @@ from __future__ import annotations -from typing import Dict, List, Union, Iterable -from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict +from typing_extensions import Required, Annotated, TypedDict from .._utils import PropertyInfo -from .shared_params.sampling_params import SamplingParams -from .rest_api_execution_config_param import RestAPIExecutionConfigParam +from .shared_params.agent_config import AgentConfig -__all__ = [ - "AgentCreateParams", - "AgentConfig", - "AgentConfigTool", - "AgentConfigToolSearchToolDefinition", - "AgentConfigToolWolframAlphaToolDefinition", - "AgentConfigToolPhotogenToolDefinition", - "AgentConfigToolCodeInterpreterToolDefinition", - "AgentConfigToolFunctionCallToolDefinition", - "AgentConfigToolFunctionCallToolDefinitionParameters", - "AgentConfigToolMemoryToolDefinition", - "AgentConfigToolMemoryToolDefinitionMemoryBankConfig", - "AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember0", - "AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember1", - "AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember2", - "AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember3", - "AgentConfigToolMemoryToolDefinitionQueryGeneratorConfig", - "AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigUnionMember0", - "AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigUnionMember1", - "AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigType", -] +__all__ = ["AgentCreateParams"] class AgentCreateParams(TypedDict, total=False): agent_config: Required[AgentConfig] x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] - - -class AgentConfigToolSearchToolDefinition(TypedDict, total=False): - api_key: Required[str] - - engine: Required[Literal["bing", "brave"]] - - type: Required[Literal["brave_search"]] - - input_shields: List[str] - - output_shields: List[str] - - remote_execution: RestAPIExecutionConfigParam - - -class AgentConfigToolWolframAlphaToolDefinition(TypedDict, total=False): - api_key: Required[str] - - type: Required[Literal["wolfram_alpha"]] - - input_shields: List[str] - - output_shields: List[str] - - remote_execution: RestAPIExecutionConfigParam - - -class AgentConfigToolPhotogenToolDefinition(TypedDict, total=False): - type: Required[Literal["photogen"]] - - input_shields: List[str] - - output_shields: List[str] - - remote_execution: RestAPIExecutionConfigParam - - -class AgentConfigToolCodeInterpreterToolDefinition(TypedDict, total=False): - enable_inline_code_execution: Required[bool] - - type: Required[Literal["code_interpreter"]] - - input_shields: List[str] - - output_shields: List[str] - - remote_execution: RestAPIExecutionConfigParam - - -class AgentConfigToolFunctionCallToolDefinitionParameters(TypedDict, total=False): - param_type: Required[str] - - default: Union[bool, float, str, Iterable[object], object, None] - - description: str - - required: bool - - -class AgentConfigToolFunctionCallToolDefinition(TypedDict, total=False): - description: Required[str] - - function_name: Required[str] - - parameters: Required[Dict[str, AgentConfigToolFunctionCallToolDefinitionParameters]] - - type: Required[Literal["function_call"]] - - input_shields: List[str] - - output_shields: List[str] - - remote_execution: RestAPIExecutionConfigParam - - -class AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember0(TypedDict, total=False): - bank_id: Required[str] - - type: Required[Literal["vector"]] - - -class AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember1(TypedDict, total=False): - bank_id: Required[str] - - keys: Required[List[str]] - - type: Required[Literal["keyvalue"]] - - -class AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember2(TypedDict, total=False): - bank_id: Required[str] - - type: Required[Literal["keyword"]] - - -class AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember3(TypedDict, total=False): - bank_id: Required[str] - - entities: Required[List[str]] - - type: Required[Literal["graph"]] - - -AgentConfigToolMemoryToolDefinitionMemoryBankConfig: TypeAlias = Union[ - AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember0, - AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember1, - AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember2, - AgentConfigToolMemoryToolDefinitionMemoryBankConfigUnionMember3, -] - - -class AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigUnionMember0(TypedDict, total=False): - sep: Required[str] - - type: Required[Literal["default"]] - - -class AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigUnionMember1(TypedDict, total=False): - model: Required[str] - - template: Required[str] - - type: Required[Literal["llm"]] - - -class AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigType(TypedDict, total=False): - type: Required[Literal["custom"]] - - -AgentConfigToolMemoryToolDefinitionQueryGeneratorConfig: TypeAlias = Union[ - AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigUnionMember0, - AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigUnionMember1, - AgentConfigToolMemoryToolDefinitionQueryGeneratorConfigType, -] - - -class AgentConfigToolMemoryToolDefinition(TypedDict, total=False): - max_chunks: Required[int] - - max_tokens_in_context: Required[int] - - memory_bank_configs: Required[Iterable[AgentConfigToolMemoryToolDefinitionMemoryBankConfig]] - - query_generator_config: Required[AgentConfigToolMemoryToolDefinitionQueryGeneratorConfig] - - type: Required[Literal["memory"]] - - input_shields: List[str] - - output_shields: List[str] - - -AgentConfigTool: TypeAlias = Union[ - AgentConfigToolSearchToolDefinition, - AgentConfigToolWolframAlphaToolDefinition, - AgentConfigToolPhotogenToolDefinition, - AgentConfigToolCodeInterpreterToolDefinition, - AgentConfigToolFunctionCallToolDefinition, - AgentConfigToolMemoryToolDefinition, -] - - -class AgentConfig(TypedDict, total=False): - enable_session_persistence: Required[bool] - - instructions: Required[str] - - max_infer_iters: Required[int] - - model: Required[str] - - input_shields: List[str] - - output_shields: List[str] - - sampling_params: SamplingParams - - tool_choice: Literal["auto", "required"] - - tool_prompt_format: Literal["json", "function_tag", "python_list"] - """ - `json` -- Refers to the json format for calling tools. The json format takes the - form like { "type": "function", "function" : { "name": "function_name", - "description": "function_description", "parameters": {...} } } - - `function_tag` -- This is an example of how you could define your own user - defined format for making tool calls. The function_tag format looks like this, - (parameters) - - The detailed prompts for each of these formats are added to llama cli - """ - - tools: Iterable[AgentConfigTool] diff --git a/src/llama_stack_client/types/agents/turn_create_response.py b/src/llama_stack_client/types/agents/turn_create_response.py index 596a3e5e..4fba0ca0 100644 --- a/src/llama_stack_client/types/agents/turn_create_response.py +++ b/src/llama_stack_client/types/agents/turn_create_response.py @@ -15,20 +15,21 @@ __all__ = [ "TurnCreateResponse", - "Event", - "EventPayload", - "EventPayloadAgentTurnResponseStepStartPayload", - "EventPayloadAgentTurnResponseStepProgressPayload", - "EventPayloadAgentTurnResponseStepProgressPayloadToolCallDelta", - "EventPayloadAgentTurnResponseStepProgressPayloadToolCallDeltaContent", - "EventPayloadAgentTurnResponseStepCompletePayload", - "EventPayloadAgentTurnResponseStepCompletePayloadStepDetails", - "EventPayloadAgentTurnResponseTurnStartPayload", - "EventPayloadAgentTurnResponseTurnCompletePayload", + "AgentTurnResponseStreamChunk", + "AgentTurnResponseStreamChunkEvent", + "AgentTurnResponseStreamChunkEventPayload", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepStartPayload", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayload", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayloadToolCallDelta", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayloadToolCallDeltaContent", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepCompletePayload", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepCompletePayloadStepDetails", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseTurnStartPayload", + "AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseTurnCompletePayload", ] -class EventPayloadAgentTurnResponseStepStartPayload(BaseModel): +class AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepStartPayload(BaseModel): event_type: Literal["step_start"] step_id: str @@ -38,16 +39,18 @@ class EventPayloadAgentTurnResponseStepStartPayload(BaseModel): metadata: Optional[Dict[str, Union[bool, float, str, List[object], object, None]]] = None -EventPayloadAgentTurnResponseStepProgressPayloadToolCallDeltaContent: TypeAlias = Union[str, ToolCall] +AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayloadToolCallDeltaContent: TypeAlias = Union[ + str, ToolCall +] -class EventPayloadAgentTurnResponseStepProgressPayloadToolCallDelta(BaseModel): - content: EventPayloadAgentTurnResponseStepProgressPayloadToolCallDeltaContent +class AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayloadToolCallDelta(BaseModel): + content: AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayloadToolCallDeltaContent parse_status: Literal["started", "in_progress", "failure", "success"] -class EventPayloadAgentTurnResponseStepProgressPayload(BaseModel): +class AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayload(BaseModel): event_type: Literal["step_progress"] step_id: str @@ -56,48 +59,53 @@ class EventPayloadAgentTurnResponseStepProgressPayload(BaseModel): text_delta_model_response: Optional[str] = FieldInfo(alias="model_response_text_delta", default=None) - tool_call_delta: Optional[EventPayloadAgentTurnResponseStepProgressPayloadToolCallDelta] = None + tool_call_delta: Optional[ + AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayloadToolCallDelta + ] = None tool_response_text_delta: Optional[str] = None -EventPayloadAgentTurnResponseStepCompletePayloadStepDetails: TypeAlias = Union[ +AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepCompletePayloadStepDetails: TypeAlias = Union[ InferenceStep, ToolExecutionStep, ShieldCallStep, MemoryRetrievalStep ] -class EventPayloadAgentTurnResponseStepCompletePayload(BaseModel): +class AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepCompletePayload(BaseModel): event_type: Literal["step_complete"] - step_details: EventPayloadAgentTurnResponseStepCompletePayloadStepDetails + step_details: AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepCompletePayloadStepDetails step_type: Literal["inference", "tool_execution", "shield_call", "memory_retrieval"] -class EventPayloadAgentTurnResponseTurnStartPayload(BaseModel): +class AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseTurnStartPayload(BaseModel): event_type: Literal["turn_start"] turn_id: str -class EventPayloadAgentTurnResponseTurnCompletePayload(BaseModel): +class AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseTurnCompletePayload(BaseModel): event_type: Literal["turn_complete"] turn: Turn -EventPayload: TypeAlias = Union[ - EventPayloadAgentTurnResponseStepStartPayload, - EventPayloadAgentTurnResponseStepProgressPayload, - EventPayloadAgentTurnResponseStepCompletePayload, - EventPayloadAgentTurnResponseTurnStartPayload, - EventPayloadAgentTurnResponseTurnCompletePayload, +AgentTurnResponseStreamChunkEventPayload: TypeAlias = Union[ + AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepStartPayload, + AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepProgressPayload, + AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseStepCompletePayload, + AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseTurnStartPayload, + AgentTurnResponseStreamChunkEventPayloadAgentTurnResponseTurnCompletePayload, ] -class Event(BaseModel): - payload: EventPayload +class AgentTurnResponseStreamChunkEvent(BaseModel): + payload: AgentTurnResponseStreamChunkEventPayload + + +class AgentTurnResponseStreamChunk(BaseModel): + event: AgentTurnResponseStreamChunkEvent -class TurnCreateResponse(BaseModel): - event: Event +TurnCreateResponse: TypeAlias = Union[Turn, AgentTurnResponseStreamChunk] diff --git a/src/llama_stack_client/types/batch_inference_chat_completion_params.py b/src/llama_stack_client/types/batch_inference_chat_completion_params.py index 6b9ca80c..0bd88aa7 100644 --- a/src/llama_stack_client/types/batch_inference_chat_completion_params.py +++ b/src/llama_stack_client/types/batch_inference_chat_completion_params.py @@ -10,9 +10,10 @@ from .shared_params.system_message import SystemMessage from .shared_params.sampling_params import SamplingParams from .shared_params.completion_message import CompletionMessage +from .shared_params.tool_param_definition import ToolParamDefinition from .shared_params.tool_response_message import ToolResponseMessage -__all__ = ["BatchInferenceChatCompletionParams", "MessagesBatch", "Logprobs", "Tool", "ToolParameters"] +__all__ = ["BatchInferenceChatCompletionParams", "MessagesBatch", "Logprobs", "Tool"] class BatchInferenceChatCompletionParams(TypedDict, total=False): @@ -51,19 +52,9 @@ class Logprobs(TypedDict, total=False): top_k: int -class ToolParameters(TypedDict, total=False): - param_type: Required[str] - - default: Union[bool, float, str, Iterable[object], object, None] - - description: str - - required: bool - - class Tool(TypedDict, total=False): tool_name: Required[Union[Literal["brave_search", "wolfram_alpha", "photogen", "code_interpreter"], str]] description: str - parameters: Dict[str, ToolParameters] + parameters: Dict[str, ToolParamDefinition] diff --git a/src/llama_stack_client/types/batch_inference_completion_params.py b/src/llama_stack_client/types/batch_inference_completion_params.py index 0658c2c6..1d87e14b 100644 --- a/src/llama_stack_client/types/batch_inference_completion_params.py +++ b/src/llama_stack_client/types/batch_inference_completion_params.py @@ -9,7 +9,7 @@ from .shared_params.image_media import ImageMedia from .shared_params.sampling_params import SamplingParams -__all__ = ["BatchInferenceCompletionParams", "ContentBatch", "ContentBatchUnionMember2", "Logprobs"] +__all__ = ["BatchInferenceCompletionParams", "ContentBatch", "ContentBatchImageMediaArray", "Logprobs"] class BatchInferenceCompletionParams(TypedDict, total=False): @@ -24,9 +24,9 @@ class BatchInferenceCompletionParams(TypedDict, total=False): x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] -ContentBatchUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentBatchImageMediaArray: TypeAlias = Union[str, ImageMedia] -ContentBatch: TypeAlias = Union[str, ImageMedia, List[ContentBatchUnionMember2]] +ContentBatch: TypeAlias = Union[str, ImageMedia, List[ContentBatchImageMediaArray]] class Logprobs(TypedDict, total=False): diff --git a/src/llama_stack_client/types/dataset_list_response.py b/src/llama_stack_client/types/dataset_list_response.py index 993bf13c..75647f06 100644 --- a/src/llama_stack_client/types/dataset_list_response.py +++ b/src/llama_stack_client/types/dataset_list_response.py @@ -5,24 +5,73 @@ from .._models import BaseModel -__all__ = ["DatasetListResponse", "DatasetSchema", "DatasetSchemaType"] +__all__ = [ + "DatasetListResponse", + "DatasetSchema", + "DatasetSchemaDsString", + "DatasetSchemaDsNumber", + "DatasetSchemaDsBoolean", + "DatasetSchemaDsArray", + "DatasetSchemaDsObject", + "DatasetSchemaDsJson", + "DatasetSchemaDsUnion", + "DatasetSchemaDsChatCompletionInput", + "DatasetSchemaDsCompletionInput", + "DatasetSchemaDsAgentTurnInput", +] -class DatasetSchemaType(BaseModel): +class DatasetSchemaDsString(BaseModel): type: Literal["string"] +class DatasetSchemaDsNumber(BaseModel): + type: Literal["number"] + + +class DatasetSchemaDsBoolean(BaseModel): + type: Literal["boolean"] + + +class DatasetSchemaDsArray(BaseModel): + type: Literal["array"] + + +class DatasetSchemaDsObject(BaseModel): + type: Literal["object"] + + +class DatasetSchemaDsJson(BaseModel): + type: Literal["json"] + + +class DatasetSchemaDsUnion(BaseModel): + type: Literal["union"] + + +class DatasetSchemaDsChatCompletionInput(BaseModel): + type: Literal["chat_completion_input"] + + +class DatasetSchemaDsCompletionInput(BaseModel): + type: Literal["completion_input"] + + +class DatasetSchemaDsAgentTurnInput(BaseModel): + type: Literal["agent_turn_input"] + + DatasetSchema: TypeAlias = Union[ - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, + DatasetSchemaDsString, + DatasetSchemaDsNumber, + DatasetSchemaDsBoolean, + DatasetSchemaDsArray, + DatasetSchemaDsObject, + DatasetSchemaDsJson, + DatasetSchemaDsUnion, + DatasetSchemaDsChatCompletionInput, + DatasetSchemaDsCompletionInput, + DatasetSchemaDsAgentTurnInput, ] @@ -35,4 +84,6 @@ class DatasetListResponse(BaseModel): provider_id: str + type: Literal["dataset"] + url: str diff --git a/src/llama_stack_client/types/dataset_register_params.py b/src/llama_stack_client/types/dataset_register_params.py index c7228c18..b2bad32e 100644 --- a/src/llama_stack_client/types/dataset_register_params.py +++ b/src/llama_stack_client/types/dataset_register_params.py @@ -7,7 +7,21 @@ from .._utils import PropertyInfo -__all__ = ["DatasetRegisterParams", "DatasetDef", "DatasetDefDatasetSchema", "DatasetDefDatasetSchemaType"] +__all__ = [ + "DatasetRegisterParams", + "DatasetDef", + "DatasetDefDatasetSchema", + "DatasetDefDatasetSchemaDsString", + "DatasetDefDatasetSchemaDsNumber", + "DatasetDefDatasetSchemaDsBoolean", + "DatasetDefDatasetSchemaDsArray", + "DatasetDefDatasetSchemaDsObject", + "DatasetDefDatasetSchemaDsJson", + "DatasetDefDatasetSchemaDsUnion", + "DatasetDefDatasetSchemaDsChatCompletionInput", + "DatasetDefDatasetSchemaDsCompletionInput", + "DatasetDefDatasetSchemaDsAgentTurnInput", +] class DatasetRegisterParams(TypedDict, total=False): @@ -16,21 +30,57 @@ class DatasetRegisterParams(TypedDict, total=False): x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] -class DatasetDefDatasetSchemaType(TypedDict, total=False): +class DatasetDefDatasetSchemaDsString(TypedDict, total=False): type: Required[Literal["string"]] +class DatasetDefDatasetSchemaDsNumber(TypedDict, total=False): + type: Required[Literal["number"]] + + +class DatasetDefDatasetSchemaDsBoolean(TypedDict, total=False): + type: Required[Literal["boolean"]] + + +class DatasetDefDatasetSchemaDsArray(TypedDict, total=False): + type: Required[Literal["array"]] + + +class DatasetDefDatasetSchemaDsObject(TypedDict, total=False): + type: Required[Literal["object"]] + + +class DatasetDefDatasetSchemaDsJson(TypedDict, total=False): + type: Required[Literal["json"]] + + +class DatasetDefDatasetSchemaDsUnion(TypedDict, total=False): + type: Required[Literal["union"]] + + +class DatasetDefDatasetSchemaDsChatCompletionInput(TypedDict, total=False): + type: Required[Literal["chat_completion_input"]] + + +class DatasetDefDatasetSchemaDsCompletionInput(TypedDict, total=False): + type: Required[Literal["completion_input"]] + + +class DatasetDefDatasetSchemaDsAgentTurnInput(TypedDict, total=False): + type: Required[Literal["agent_turn_input"]] + + DatasetDefDatasetSchema: TypeAlias = Union[ - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, - DatasetDefDatasetSchemaType, + DatasetDefDatasetSchemaDsString, + DatasetDefDatasetSchemaDsNumber, + DatasetDefDatasetSchemaDsBoolean, + DatasetDefDatasetSchemaDsArray, + DatasetDefDatasetSchemaDsObject, + DatasetDefDatasetSchemaDsJson, + DatasetDefDatasetSchemaDsUnion, + DatasetDefDatasetSchemaDsChatCompletionInput, + DatasetDefDatasetSchemaDsCompletionInput, + DatasetDefDatasetSchemaDsAgentTurnInput, ] @@ -43,4 +93,6 @@ class DatasetDef(TypedDict, total=False): provider_id: Required[str] + type: Required[Literal["dataset"]] + url: Required[str] diff --git a/src/llama_stack_client/types/dataset_retrieve_response.py b/src/llama_stack_client/types/dataset_retrieve_response.py index 2c7400d1..406ec3f7 100644 --- a/src/llama_stack_client/types/dataset_retrieve_response.py +++ b/src/llama_stack_client/types/dataset_retrieve_response.py @@ -5,24 +5,73 @@ from .._models import BaseModel -__all__ = ["DatasetRetrieveResponse", "DatasetSchema", "DatasetSchemaType"] +__all__ = [ + "DatasetRetrieveResponse", + "DatasetSchema", + "DatasetSchemaDsString", + "DatasetSchemaDsNumber", + "DatasetSchemaDsBoolean", + "DatasetSchemaDsArray", + "DatasetSchemaDsObject", + "DatasetSchemaDsJson", + "DatasetSchemaDsUnion", + "DatasetSchemaDsChatCompletionInput", + "DatasetSchemaDsCompletionInput", + "DatasetSchemaDsAgentTurnInput", +] -class DatasetSchemaType(BaseModel): +class DatasetSchemaDsString(BaseModel): type: Literal["string"] +class DatasetSchemaDsNumber(BaseModel): + type: Literal["number"] + + +class DatasetSchemaDsBoolean(BaseModel): + type: Literal["boolean"] + + +class DatasetSchemaDsArray(BaseModel): + type: Literal["array"] + + +class DatasetSchemaDsObject(BaseModel): + type: Literal["object"] + + +class DatasetSchemaDsJson(BaseModel): + type: Literal["json"] + + +class DatasetSchemaDsUnion(BaseModel): + type: Literal["union"] + + +class DatasetSchemaDsChatCompletionInput(BaseModel): + type: Literal["chat_completion_input"] + + +class DatasetSchemaDsCompletionInput(BaseModel): + type: Literal["completion_input"] + + +class DatasetSchemaDsAgentTurnInput(BaseModel): + type: Literal["agent_turn_input"] + + DatasetSchema: TypeAlias = Union[ - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, - DatasetSchemaType, + DatasetSchemaDsString, + DatasetSchemaDsNumber, + DatasetSchemaDsBoolean, + DatasetSchemaDsArray, + DatasetSchemaDsObject, + DatasetSchemaDsJson, + DatasetSchemaDsUnion, + DatasetSchemaDsChatCompletionInput, + DatasetSchemaDsCompletionInput, + DatasetSchemaDsAgentTurnInput, ] @@ -35,4 +84,6 @@ class DatasetRetrieveResponse(BaseModel): provider_id: str + type: Literal["dataset"] + url: str diff --git a/src/llama_stack_client/types/eval/__init__.py b/src/llama_stack_client/types/eval/__init__.py index 95e97450..cf800c42 100644 --- a/src/llama_stack_client/types/eval/__init__.py +++ b/src/llama_stack_client/types/eval/__init__.py @@ -2,8 +2,8 @@ from __future__ import annotations -from .job_status import JobStatus as JobStatus from .job_cancel_params import JobCancelParams as JobCancelParams -from .job_result_params import JobResultParams as JobResultParams from .job_status_params import JobStatusParams as JobStatusParams -from .job_result_response import JobResultResponse as JobResultResponse +from .job_retrieve_params import JobRetrieveParams as JobRetrieveParams +from .job_status_response import JobStatusResponse as JobStatusResponse +from .job_retrieve_response import JobRetrieveResponse as JobRetrieveResponse diff --git a/src/llama_stack_client/types/eval/job_cancel_params.py b/src/llama_stack_client/types/eval/job_cancel_params.py index 337f6803..699ffde4 100644 --- a/src/llama_stack_client/types/eval/job_cancel_params.py +++ b/src/llama_stack_client/types/eval/job_cancel_params.py @@ -12,4 +12,6 @@ class JobCancelParams(TypedDict, total=False): job_id: Required[str] + task_id: Required[str] + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/eval/job_retrieve_params.py b/src/llama_stack_client/types/eval/job_retrieve_params.py new file mode 100644 index 00000000..2c26fce2 --- /dev/null +++ b/src/llama_stack_client/types/eval/job_retrieve_params.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, Annotated, TypedDict + +from ..._utils import PropertyInfo + +__all__ = ["JobRetrieveParams"] + + +class JobRetrieveParams(TypedDict, total=False): + job_id: Required[str] + + task_id: Required[str] + + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/eval/job_retrieve_response.py b/src/llama_stack_client/types/eval/job_retrieve_response.py new file mode 100644 index 00000000..6d962de6 --- /dev/null +++ b/src/llama_stack_client/types/eval/job_retrieve_response.py @@ -0,0 +1,14 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Union + +from ..._models import BaseModel +from ..shared.scoring_result import ScoringResult + +__all__ = ["JobRetrieveResponse"] + + +class JobRetrieveResponse(BaseModel): + generations: List[Dict[str, Union[bool, float, str, List[object], object, None]]] + + scores: Dict[str, ScoringResult] diff --git a/src/llama_stack_client/types/eval/job_status_params.py b/src/llama_stack_client/types/eval/job_status_params.py index 01070e2a..55877faf 100644 --- a/src/llama_stack_client/types/eval/job_status_params.py +++ b/src/llama_stack_client/types/eval/job_status_params.py @@ -12,4 +12,6 @@ class JobStatusParams(TypedDict, total=False): job_id: Required[str] + task_id: Required[str] + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/eval/job_status_response.py b/src/llama_stack_client/types/eval/job_status_response.py new file mode 100644 index 00000000..301913b0 --- /dev/null +++ b/src/llama_stack_client/types/eval/job_status_response.py @@ -0,0 +1,8 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal, TypeAlias + +__all__ = ["JobStatusResponse"] + +JobStatusResponse: TypeAlias = Optional[Literal["completed", "in_progress"]] diff --git a/src/llama_stack_client/types/eval_evaluate_rows_params.py b/src/llama_stack_client/types/eval_evaluate_rows_params.py new file mode 100644 index 00000000..d09541fc --- /dev/null +++ b/src/llama_stack_client/types/eval_evaluate_rows_params.py @@ -0,0 +1,125 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict, List, Union, Iterable +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict + +from .._utils import PropertyInfo +from .shared_params.agent_config import AgentConfig +from .shared_params.system_message import SystemMessage +from .shared_params.sampling_params import SamplingParams + +__all__ = [ + "EvalEvaluateRowsParams", + "TaskConfig", + "TaskConfigBenchmarkEvalTaskConfig", + "TaskConfigBenchmarkEvalTaskConfigEvalCandidate", + "TaskConfigBenchmarkEvalTaskConfigEvalCandidateModelCandidate", + "TaskConfigBenchmarkEvalTaskConfigEvalCandidateAgentCandidate", + "TaskConfigAppEvalTaskConfig", + "TaskConfigAppEvalTaskConfigEvalCandidate", + "TaskConfigAppEvalTaskConfigEvalCandidateModelCandidate", + "TaskConfigAppEvalTaskConfigEvalCandidateAgentCandidate", + "TaskConfigAppEvalTaskConfigScoringParams", + "TaskConfigAppEvalTaskConfigScoringParamsLlmAsJudgeScoringFnParams", + "TaskConfigAppEvalTaskConfigScoringParamsRegexParserScoringFnParams", +] + + +class EvalEvaluateRowsParams(TypedDict, total=False): + input_rows: Required[Iterable[Dict[str, Union[bool, float, str, Iterable[object], object, None]]]] + + scoring_functions: Required[List[str]] + + task_config: Required[TaskConfig] + + task_id: Required[str] + + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] + + +class TaskConfigBenchmarkEvalTaskConfigEvalCandidateModelCandidate(TypedDict, total=False): + model: Required[str] + + sampling_params: Required[SamplingParams] + + type: Required[Literal["model"]] + + system_message: SystemMessage + + +class TaskConfigBenchmarkEvalTaskConfigEvalCandidateAgentCandidate(TypedDict, total=False): + config: Required[AgentConfig] + + type: Required[Literal["agent"]] + + +TaskConfigBenchmarkEvalTaskConfigEvalCandidate: TypeAlias = Union[ + TaskConfigBenchmarkEvalTaskConfigEvalCandidateModelCandidate, + TaskConfigBenchmarkEvalTaskConfigEvalCandidateAgentCandidate, +] + + +class TaskConfigBenchmarkEvalTaskConfig(TypedDict, total=False): + eval_candidate: Required[TaskConfigBenchmarkEvalTaskConfigEvalCandidate] + + type: Required[Literal["benchmark"]] + + num_examples: int + + +class TaskConfigAppEvalTaskConfigEvalCandidateModelCandidate(TypedDict, total=False): + model: Required[str] + + sampling_params: Required[SamplingParams] + + type: Required[Literal["model"]] + + system_message: SystemMessage + + +class TaskConfigAppEvalTaskConfigEvalCandidateAgentCandidate(TypedDict, total=False): + config: Required[AgentConfig] + + type: Required[Literal["agent"]] + + +TaskConfigAppEvalTaskConfigEvalCandidate: TypeAlias = Union[ + TaskConfigAppEvalTaskConfigEvalCandidateModelCandidate, TaskConfigAppEvalTaskConfigEvalCandidateAgentCandidate +] + + +class TaskConfigAppEvalTaskConfigScoringParamsLlmAsJudgeScoringFnParams(TypedDict, total=False): + judge_model: Required[str] + + type: Required[Literal["llm_as_judge"]] + + judge_score_regexes: List[str] + + prompt_template: str + + +class TaskConfigAppEvalTaskConfigScoringParamsRegexParserScoringFnParams(TypedDict, total=False): + type: Required[Literal["regex_parser"]] + + parsing_regexes: List[str] + + +TaskConfigAppEvalTaskConfigScoringParams: TypeAlias = Union[ + TaskConfigAppEvalTaskConfigScoringParamsLlmAsJudgeScoringFnParams, + TaskConfigAppEvalTaskConfigScoringParamsRegexParserScoringFnParams, +] + + +class TaskConfigAppEvalTaskConfig(TypedDict, total=False): + eval_candidate: Required[TaskConfigAppEvalTaskConfigEvalCandidate] + + scoring_params: Required[Dict[str, TaskConfigAppEvalTaskConfigScoringParams]] + + type: Required[Literal["app"]] + + num_examples: int + + +TaskConfig: TypeAlias = Union[TaskConfigBenchmarkEvalTaskConfig, TaskConfigAppEvalTaskConfig] diff --git a/src/llama_stack_client/types/eval_evaluate_rows_response.py b/src/llama_stack_client/types/eval_evaluate_rows_response.py new file mode 100644 index 00000000..98fcecfd --- /dev/null +++ b/src/llama_stack_client/types/eval_evaluate_rows_response.py @@ -0,0 +1,14 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Union + +from .._models import BaseModel +from .shared.scoring_result import ScoringResult + +__all__ = ["EvalEvaluateRowsResponse"] + + +class EvalEvaluateRowsResponse(BaseModel): + generations: List[Dict[str, Union[bool, float, str, List[object], object, None]]] + + scores: Dict[str, ScoringResult] diff --git a/src/llama_stack_client/types/eval_run_eval_params.py b/src/llama_stack_client/types/eval_run_eval_params.py new file mode 100644 index 00000000..959dde8f --- /dev/null +++ b/src/llama_stack_client/types/eval_run_eval_params.py @@ -0,0 +1,121 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict, List, Union +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict + +from .._utils import PropertyInfo +from .shared_params.agent_config import AgentConfig +from .shared_params.system_message import SystemMessage +from .shared_params.sampling_params import SamplingParams + +__all__ = [ + "EvalRunEvalParams", + "TaskConfig", + "TaskConfigBenchmarkEvalTaskConfig", + "TaskConfigBenchmarkEvalTaskConfigEvalCandidate", + "TaskConfigBenchmarkEvalTaskConfigEvalCandidateModelCandidate", + "TaskConfigBenchmarkEvalTaskConfigEvalCandidateAgentCandidate", + "TaskConfigAppEvalTaskConfig", + "TaskConfigAppEvalTaskConfigEvalCandidate", + "TaskConfigAppEvalTaskConfigEvalCandidateModelCandidate", + "TaskConfigAppEvalTaskConfigEvalCandidateAgentCandidate", + "TaskConfigAppEvalTaskConfigScoringParams", + "TaskConfigAppEvalTaskConfigScoringParamsLlmAsJudgeScoringFnParams", + "TaskConfigAppEvalTaskConfigScoringParamsRegexParserScoringFnParams", +] + + +class EvalRunEvalParams(TypedDict, total=False): + task_config: Required[TaskConfig] + + task_id: Required[str] + + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] + + +class TaskConfigBenchmarkEvalTaskConfigEvalCandidateModelCandidate(TypedDict, total=False): + model: Required[str] + + sampling_params: Required[SamplingParams] + + type: Required[Literal["model"]] + + system_message: SystemMessage + + +class TaskConfigBenchmarkEvalTaskConfigEvalCandidateAgentCandidate(TypedDict, total=False): + config: Required[AgentConfig] + + type: Required[Literal["agent"]] + + +TaskConfigBenchmarkEvalTaskConfigEvalCandidate: TypeAlias = Union[ + TaskConfigBenchmarkEvalTaskConfigEvalCandidateModelCandidate, + TaskConfigBenchmarkEvalTaskConfigEvalCandidateAgentCandidate, +] + + +class TaskConfigBenchmarkEvalTaskConfig(TypedDict, total=False): + eval_candidate: Required[TaskConfigBenchmarkEvalTaskConfigEvalCandidate] + + type: Required[Literal["benchmark"]] + + num_examples: int + + +class TaskConfigAppEvalTaskConfigEvalCandidateModelCandidate(TypedDict, total=False): + model: Required[str] + + sampling_params: Required[SamplingParams] + + type: Required[Literal["model"]] + + system_message: SystemMessage + + +class TaskConfigAppEvalTaskConfigEvalCandidateAgentCandidate(TypedDict, total=False): + config: Required[AgentConfig] + + type: Required[Literal["agent"]] + + +TaskConfigAppEvalTaskConfigEvalCandidate: TypeAlias = Union[ + TaskConfigAppEvalTaskConfigEvalCandidateModelCandidate, TaskConfigAppEvalTaskConfigEvalCandidateAgentCandidate +] + + +class TaskConfigAppEvalTaskConfigScoringParamsLlmAsJudgeScoringFnParams(TypedDict, total=False): + judge_model: Required[str] + + type: Required[Literal["llm_as_judge"]] + + judge_score_regexes: List[str] + + prompt_template: str + + +class TaskConfigAppEvalTaskConfigScoringParamsRegexParserScoringFnParams(TypedDict, total=False): + type: Required[Literal["regex_parser"]] + + parsing_regexes: List[str] + + +TaskConfigAppEvalTaskConfigScoringParams: TypeAlias = Union[ + TaskConfigAppEvalTaskConfigScoringParamsLlmAsJudgeScoringFnParams, + TaskConfigAppEvalTaskConfigScoringParamsRegexParserScoringFnParams, +] + + +class TaskConfigAppEvalTaskConfig(TypedDict, total=False): + eval_candidate: Required[TaskConfigAppEvalTaskConfigEvalCandidate] + + scoring_params: Required[Dict[str, TaskConfigAppEvalTaskConfigScoringParams]] + + type: Required[Literal["app"]] + + num_examples: int + + +TaskConfig: TypeAlias = Union[TaskConfigBenchmarkEvalTaskConfig, TaskConfigAppEvalTaskConfig] diff --git a/src/llama_stack_client/types/eval_run_eval_response.py b/src/llama_stack_client/types/eval_run_eval_response.py new file mode 100644 index 00000000..9ea7c6ae --- /dev/null +++ b/src/llama_stack_client/types/eval_run_eval_response.py @@ -0,0 +1,10 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + + +from .._models import BaseModel + +__all__ = ["EvalRunEvalResponse"] + + +class EvalRunEvalResponse(BaseModel): + job_id: str diff --git a/src/llama_stack_client/types/inference_chat_completion_params.py b/src/llama_stack_client/types/inference_chat_completion_params.py index d7ae066b..2f6d9e38 100644 --- a/src/llama_stack_client/types/inference_chat_completion_params.py +++ b/src/llama_stack_client/types/inference_chat_completion_params.py @@ -10,6 +10,7 @@ from .shared_params.system_message import SystemMessage from .shared_params.sampling_params import SamplingParams from .shared_params.completion_message import CompletionMessage +from .shared_params.tool_param_definition import ToolParamDefinition from .shared_params.tool_response_message import ToolResponseMessage __all__ = [ @@ -17,10 +18,9 @@ "Message", "Logprobs", "ResponseFormat", - "ResponseFormatUnionMember0", - "ResponseFormatUnionMember1", + "ResponseFormatJsonSchemaFormat", + "ResponseFormatGrammarFormat", "Tool", - "ToolParameters", ] @@ -64,29 +64,19 @@ class Logprobs(TypedDict, total=False): top_k: int -class ResponseFormatUnionMember0(TypedDict, total=False): +class ResponseFormatJsonSchemaFormat(TypedDict, total=False): json_schema: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] type: Required[Literal["json_schema"]] -class ResponseFormatUnionMember1(TypedDict, total=False): +class ResponseFormatGrammarFormat(TypedDict, total=False): bnf: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] type: Required[Literal["grammar"]] -ResponseFormat: TypeAlias = Union[ResponseFormatUnionMember0, ResponseFormatUnionMember1] - - -class ToolParameters(TypedDict, total=False): - param_type: Required[str] - - default: Union[bool, float, str, Iterable[object], object, None] - - description: str - - required: bool +ResponseFormat: TypeAlias = Union[ResponseFormatJsonSchemaFormat, ResponseFormatGrammarFormat] class Tool(TypedDict, total=False): @@ -94,4 +84,4 @@ class Tool(TypedDict, total=False): description: str - parameters: Dict[str, ToolParameters] + parameters: Dict[str, ToolParamDefinition] diff --git a/src/llama_stack_client/types/inference_completion_params.py b/src/llama_stack_client/types/inference_completion_params.py index d6f9275b..afec6c2c 100644 --- a/src/llama_stack_client/types/inference_completion_params.py +++ b/src/llama_stack_client/types/inference_completion_params.py @@ -12,11 +12,11 @@ __all__ = [ "InferenceCompletionParams", "Content", - "ContentUnionMember2", + "ContentImageMediaArray", "Logprobs", "ResponseFormat", - "ResponseFormatUnionMember0", - "ResponseFormatUnionMember1", + "ResponseFormatJsonSchemaFormat", + "ResponseFormatGrammarFormat", ] @@ -36,25 +36,25 @@ class InferenceCompletionParams(TypedDict, total=False): x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class Logprobs(TypedDict, total=False): top_k: int -class ResponseFormatUnionMember0(TypedDict, total=False): +class ResponseFormatJsonSchemaFormat(TypedDict, total=False): json_schema: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] type: Required[Literal["json_schema"]] -class ResponseFormatUnionMember1(TypedDict, total=False): +class ResponseFormatGrammarFormat(TypedDict, total=False): bnf: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] type: Required[Literal["grammar"]] -ResponseFormat: TypeAlias = Union[ResponseFormatUnionMember0, ResponseFormatUnionMember1] +ResponseFormat: TypeAlias = Union[ResponseFormatJsonSchemaFormat, ResponseFormatGrammarFormat] diff --git a/src/llama_stack_client/types/inference_embeddings_params.py b/src/llama_stack_client/types/inference_embeddings_params.py index a9fc68e9..a03dec44 100644 --- a/src/llama_stack_client/types/inference_embeddings_params.py +++ b/src/llama_stack_client/types/inference_embeddings_params.py @@ -8,7 +8,7 @@ from .._utils import PropertyInfo from .shared_params.image_media import ImageMedia -__all__ = ["InferenceEmbeddingsParams", "Content", "ContentUnionMember2"] +__all__ = ["InferenceEmbeddingsParams", "Content", "ContentImageMediaArray"] class InferenceEmbeddingsParams(TypedDict, total=False): @@ -19,6 +19,6 @@ class InferenceEmbeddingsParams(TypedDict, total=False): x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] diff --git a/src/llama_stack_client/types/memory_insert_params.py b/src/llama_stack_client/types/memory_insert_params.py index c9cb8eb3..439bc439 100644 --- a/src/llama_stack_client/types/memory_insert_params.py +++ b/src/llama_stack_client/types/memory_insert_params.py @@ -8,7 +8,7 @@ from .._utils import PropertyInfo from .shared_params.image_media import ImageMedia -__all__ = ["MemoryInsertParams", "Document", "DocumentContent", "DocumentContentUnionMember2"] +__all__ = ["MemoryInsertParams", "Document", "DocumentContent", "DocumentContentContentArray"] class MemoryInsertParams(TypedDict, total=False): @@ -21,9 +21,9 @@ class MemoryInsertParams(TypedDict, total=False): x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] -DocumentContentUnionMember2: TypeAlias = Union[str, ImageMedia] +DocumentContentContentArray: TypeAlias = Union[str, ImageMedia] -DocumentContent: TypeAlias = Union[str, ImageMedia, List[DocumentContentUnionMember2]] +DocumentContent: TypeAlias = Union[str, ImageMedia, List[DocumentContentContentArray]] class Document(TypedDict, total=False): diff --git a/src/llama_stack_client/types/memory_query_params.py b/src/llama_stack_client/types/memory_query_params.py index 4c5a5615..7393f490 100644 --- a/src/llama_stack_client/types/memory_query_params.py +++ b/src/llama_stack_client/types/memory_query_params.py @@ -8,7 +8,7 @@ from .._utils import PropertyInfo from .shared_params.image_media import ImageMedia -__all__ = ["MemoryQueryParams", "Query", "QueryUnionMember2"] +__all__ = ["MemoryQueryParams", "Query", "QueryImageMediaArray"] class MemoryQueryParams(TypedDict, total=False): @@ -21,6 +21,6 @@ class MemoryQueryParams(TypedDict, total=False): x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] -QueryUnionMember2: TypeAlias = Union[str, ImageMedia] +QueryImageMediaArray: TypeAlias = Union[str, ImageMedia] -Query: TypeAlias = Union[str, ImageMedia, List[QueryUnionMember2]] +Query: TypeAlias = Union[str, ImageMedia, List[QueryImageMediaArray]] diff --git a/src/llama_stack_client/types/memory_retrieval_step.py b/src/llama_stack_client/types/memory_retrieval_step.py index 9ffc988b..b11af7e7 100644 --- a/src/llama_stack_client/types/memory_retrieval_step.py +++ b/src/llama_stack_client/types/memory_retrieval_step.py @@ -7,11 +7,11 @@ from .._models import BaseModel from .shared.image_media import ImageMedia -__all__ = ["MemoryRetrievalStep", "InsertedContext", "InsertedContextUnionMember2"] +__all__ = ["MemoryRetrievalStep", "InsertedContext", "InsertedContextImageMediaArray"] -InsertedContextUnionMember2: TypeAlias = Union[str, ImageMedia] +InsertedContextImageMediaArray: TypeAlias = Union[str, ImageMedia] -InsertedContext: TypeAlias = Union[str, ImageMedia, List[InsertedContextUnionMember2]] +InsertedContext: TypeAlias = Union[str, ImageMedia, List[InsertedContextImageMediaArray]] class MemoryRetrievalStep(BaseModel): diff --git a/src/llama_stack_client/types/model_def_with_provider.py b/src/llama_stack_client/types/model_def_with_provider.py index 45771b92..8358de50 100644 --- a/src/llama_stack_client/types/model_def_with_provider.py +++ b/src/llama_stack_client/types/model_def_with_provider.py @@ -1,6 +1,7 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import Dict, List, Union +from typing_extensions import Literal from .._models import BaseModel @@ -15,3 +16,5 @@ class ModelDefWithProvider(BaseModel): metadata: Dict[str, Union[bool, float, str, List[object], object, None]] provider_id: str + + type: Literal["model"] diff --git a/src/llama_stack_client/types/model_def_with_provider_param.py b/src/llama_stack_client/types/model_def_with_provider_param.py index 718a9c48..d145dcd7 100644 --- a/src/llama_stack_client/types/model_def_with_provider_param.py +++ b/src/llama_stack_client/types/model_def_with_provider_param.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Dict, Union, Iterable -from typing_extensions import Required, TypedDict +from typing_extensions import Literal, Required, TypedDict __all__ = ["ModelDefWithProviderParam"] @@ -16,3 +16,5 @@ class ModelDefWithProviderParam(TypedDict, total=False): metadata: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] provider_id: Required[str] + + type: Required[Literal["model"]] diff --git a/src/llama_stack_client/types/query_documents_response.py b/src/llama_stack_client/types/query_documents_response.py index 88966ad5..1638eb50 100644 --- a/src/llama_stack_client/types/query_documents_response.py +++ b/src/llama_stack_client/types/query_documents_response.py @@ -6,11 +6,11 @@ from .._models import BaseModel from .shared.image_media import ImageMedia -__all__ = ["QueryDocumentsResponse", "Chunk", "ChunkContent", "ChunkContentUnionMember2"] +__all__ = ["QueryDocumentsResponse", "Chunk", "ChunkContent", "ChunkContentImageMediaArray"] -ChunkContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ChunkContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -ChunkContent: TypeAlias = Union[str, ImageMedia, List[ChunkContentUnionMember2]] +ChunkContent: TypeAlias = Union[str, ImageMedia, List[ChunkContentImageMediaArray]] class Chunk(BaseModel): diff --git a/src/llama_stack_client/types/safety_run_shield_params.py b/src/llama_stack_client/types/safety_run_shield_params.py index 430473bd..43339f7a 100644 --- a/src/llama_stack_client/types/safety_run_shield_params.py +++ b/src/llama_stack_client/types/safety_run_shield_params.py @@ -15,12 +15,12 @@ class SafetyRunShieldParams(TypedDict, total=False): + identifier: Required[str] + messages: Required[Iterable[Message]] params: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] - shield_type: Required[str] - x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/scoring_fn_def_with_provider.py b/src/llama_stack_client/types/scoring_fn_def_with_provider.py index dd8b1c1e..61f1d71a 100644 --- a/src/llama_stack_client/types/scoring_fn_def_with_provider.py +++ b/src/llama_stack_client/types/scoring_fn_def_with_provider.py @@ -7,41 +7,14 @@ __all__ = [ "ScoringFnDefWithProvider", - "Parameter", - "ParameterType", - "ParameterTypeType", "ReturnType", "ReturnTypeType", - "Context", + "Params", + "ParamsLlmAsJudgeScoringFnParams", + "ParamsRegexParserScoringFnParams", ] -class ParameterTypeType(BaseModel): - type: Literal["string"] - - -ParameterType: TypeAlias = Union[ - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, -] - - -class Parameter(BaseModel): - name: str - - type: ParameterType - - description: Optional[str] = None - - class ReturnTypeType(BaseModel): type: Literal["string"] @@ -60,25 +33,36 @@ class ReturnTypeType(BaseModel): ] -class Context(BaseModel): +class ParamsLlmAsJudgeScoringFnParams(BaseModel): judge_model: str - judge_score_regex: Optional[List[str]] = None + type: Literal["llm_as_judge"] + + judge_score_regexes: Optional[List[str]] = None prompt_template: Optional[str] = None +class ParamsRegexParserScoringFnParams(BaseModel): + type: Literal["regex_parser"] + + parsing_regexes: Optional[List[str]] = None + + +Params: TypeAlias = Union[ParamsLlmAsJudgeScoringFnParams, ParamsRegexParserScoringFnParams] + + class ScoringFnDefWithProvider(BaseModel): identifier: str metadata: Dict[str, Union[bool, float, str, List[object], object, None]] - parameters: List[Parameter] - provider_id: str return_type: ReturnType - context: Optional[Context] = None + type: Literal["scoring_fn"] description: Optional[str] = None + + params: Optional[Params] = None diff --git a/src/llama_stack_client/types/scoring_fn_def_with_provider_param.py b/src/llama_stack_client/types/scoring_fn_def_with_provider_param.py index e6f8d4fa..94a14fef 100644 --- a/src/llama_stack_client/types/scoring_fn_def_with_provider_param.py +++ b/src/llama_stack_client/types/scoring_fn_def_with_provider_param.py @@ -7,41 +7,14 @@ __all__ = [ "ScoringFnDefWithProviderParam", - "Parameter", - "ParameterType", - "ParameterTypeType", "ReturnType", "ReturnTypeType", - "Context", + "Params", + "ParamsLlmAsJudgeScoringFnParams", + "ParamsRegexParserScoringFnParams", ] -class ParameterTypeType(TypedDict, total=False): - type: Required[Literal["string"]] - - -ParameterType: TypeAlias = Union[ - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, - ParameterTypeType, -] - - -class Parameter(TypedDict, total=False): - name: Required[str] - - type: Required[ParameterType] - - description: str - - class ReturnTypeType(TypedDict, total=False): type: Required[Literal["string"]] @@ -60,25 +33,36 @@ class ReturnTypeType(TypedDict, total=False): ] -class Context(TypedDict, total=False): +class ParamsLlmAsJudgeScoringFnParams(TypedDict, total=False): judge_model: Required[str] - judge_score_regex: List[str] + type: Required[Literal["llm_as_judge"]] + + judge_score_regexes: List[str] prompt_template: str +class ParamsRegexParserScoringFnParams(TypedDict, total=False): + type: Required[Literal["regex_parser"]] + + parsing_regexes: List[str] + + +Params: TypeAlias = Union[ParamsLlmAsJudgeScoringFnParams, ParamsRegexParserScoringFnParams] + + class ScoringFnDefWithProviderParam(TypedDict, total=False): identifier: Required[str] metadata: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] - parameters: Required[Iterable[Parameter]] - provider_id: Required[str] return_type: Required[ReturnType] - context: Context + type: Required[Literal["scoring_fn"]] description: str + + params: Params diff --git a/src/llama_stack_client/types/scoring_score_batch_params.py b/src/llama_stack_client/types/scoring_score_batch_params.py index f23c23a4..9d318ecb 100644 --- a/src/llama_stack_client/types/scoring_score_batch_params.py +++ b/src/llama_stack_client/types/scoring_score_batch_params.py @@ -2,12 +2,17 @@ from __future__ import annotations -from typing import List -from typing_extensions import Required, Annotated, TypedDict +from typing import Dict, List, Union, Optional +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict from .._utils import PropertyInfo -__all__ = ["ScoringScoreBatchParams"] +__all__ = [ + "ScoringScoreBatchParams", + "ScoringFunctions", + "ScoringFunctionsLlmAsJudgeScoringFnParams", + "ScoringFunctionsRegexParserScoringFnParams", +] class ScoringScoreBatchParams(TypedDict, total=False): @@ -15,6 +20,27 @@ class ScoringScoreBatchParams(TypedDict, total=False): save_results_dataset: Required[bool] - scoring_functions: Required[List[str]] + scoring_functions: Required[Dict[str, Optional[ScoringFunctions]]] x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] + + +class ScoringFunctionsLlmAsJudgeScoringFnParams(TypedDict, total=False): + judge_model: Required[str] + + type: Required[Literal["llm_as_judge"]] + + judge_score_regexes: List[str] + + prompt_template: str + + +class ScoringFunctionsRegexParserScoringFnParams(TypedDict, total=False): + type: Required[Literal["regex_parser"]] + + parsing_regexes: List[str] + + +ScoringFunctions: TypeAlias = Union[ + ScoringFunctionsLlmAsJudgeScoringFnParams, ScoringFunctionsRegexParserScoringFnParams +] diff --git a/src/llama_stack_client/types/scoring_score_batch_response.py b/src/llama_stack_client/types/scoring_score_batch_response.py new file mode 100644 index 00000000..da124b1e --- /dev/null +++ b/src/llama_stack_client/types/scoring_score_batch_response.py @@ -0,0 +1,14 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, Optional + +from .._models import BaseModel +from .shared.scoring_result import ScoringResult + +__all__ = ["ScoringScoreBatchResponse"] + + +class ScoringScoreBatchResponse(BaseModel): + results: Dict[str, ScoringResult] + + dataset_id: Optional[str] = None diff --git a/src/llama_stack_client/types/scoring_score_params.py b/src/llama_stack_client/types/scoring_score_params.py index 8419b096..5522f70b 100644 --- a/src/llama_stack_client/types/scoring_score_params.py +++ b/src/llama_stack_client/types/scoring_score_params.py @@ -2,17 +2,43 @@ from __future__ import annotations -from typing import Dict, List, Union, Iterable -from typing_extensions import Required, Annotated, TypedDict +from typing import Dict, List, Union, Iterable, Optional +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict from .._utils import PropertyInfo -__all__ = ["ScoringScoreParams"] +__all__ = [ + "ScoringScoreParams", + "ScoringFunctions", + "ScoringFunctionsLlmAsJudgeScoringFnParams", + "ScoringFunctionsRegexParserScoringFnParams", +] class ScoringScoreParams(TypedDict, total=False): input_rows: Required[Iterable[Dict[str, Union[bool, float, str, Iterable[object], object, None]]]] - scoring_functions: Required[List[str]] + scoring_functions: Required[Dict[str, Optional[ScoringFunctions]]] x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] + + +class ScoringFunctionsLlmAsJudgeScoringFnParams(TypedDict, total=False): + judge_model: Required[str] + + type: Required[Literal["llm_as_judge"]] + + judge_score_regexes: List[str] + + prompt_template: str + + +class ScoringFunctionsRegexParserScoringFnParams(TypedDict, total=False): + type: Required[Literal["regex_parser"]] + + parsing_regexes: List[str] + + +ScoringFunctions: TypeAlias = Union[ + ScoringFunctionsLlmAsJudgeScoringFnParams, ScoringFunctionsRegexParserScoringFnParams +] diff --git a/src/llama_stack_client/types/scoring_score_response.py b/src/llama_stack_client/types/scoring_score_response.py new file mode 100644 index 00000000..4efc9b91 --- /dev/null +++ b/src/llama_stack_client/types/scoring_score_response.py @@ -0,0 +1,12 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict + +from .._models import BaseModel +from .shared.scoring_result import ScoringResult + +__all__ = ["ScoringScoreResponse"] + + +class ScoringScoreResponse(BaseModel): + results: Dict[str, ScoringResult] diff --git a/src/llama_stack_client/types/shared/__init__.py b/src/llama_stack_client/types/shared/__init__.py index 21cc1d96..c78df35c 100644 --- a/src/llama_stack_client/types/shared/__init__.py +++ b/src/llama_stack_client/types/shared/__init__.py @@ -3,14 +3,23 @@ from .tool_call import ToolCall as ToolCall from .attachment import Attachment as Attachment from .image_media import ImageMedia as ImageMedia +from .agent_config import AgentConfig as AgentConfig from .user_message import UserMessage as UserMessage +from .scoring_result import ScoringResult as ScoringResult from .system_message import SystemMessage as SystemMessage from .sampling_params import SamplingParams as SamplingParams from .batch_completion import BatchCompletion as BatchCompletion from .safety_violation import SafetyViolation as SafetyViolation from .completion_message import CompletionMessage as CompletionMessage from .graph_memory_bank_def import GraphMemoryBankDef as GraphMemoryBankDef +from .tool_param_definition import ToolParamDefinition as ToolParamDefinition from .tool_response_message import ToolResponseMessage as ToolResponseMessage +from .memory_tool_definition import MemoryToolDefinition as MemoryToolDefinition +from .search_tool_definition import SearchToolDefinition as SearchToolDefinition from .vector_memory_bank_def import VectorMemoryBankDef as VectorMemoryBankDef from .keyword_memory_bank_def import KeywordMemoryBankDef as KeywordMemoryBankDef +from .photogen_tool_definition import PhotogenToolDefinition as PhotogenToolDefinition from .key_value_memory_bank_def import KeyValueMemoryBankDef as KeyValueMemoryBankDef +from .function_call_tool_definition import FunctionCallToolDefinition as FunctionCallToolDefinition +from .wolfram_alpha_tool_definition import WolframAlphaToolDefinition as WolframAlphaToolDefinition +from .code_interpreter_tool_definition import CodeInterpreterToolDefinition as CodeInterpreterToolDefinition diff --git a/src/llama_stack_client/types/shared/agent_config.py b/src/llama_stack_client/types/shared/agent_config.py new file mode 100644 index 00000000..42362422 --- /dev/null +++ b/src/llama_stack_client/types/shared/agent_config.py @@ -0,0 +1,57 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal, TypeAlias + +from ..._models import BaseModel +from .sampling_params import SamplingParams +from .memory_tool_definition import MemoryToolDefinition +from .search_tool_definition import SearchToolDefinition +from .photogen_tool_definition import PhotogenToolDefinition +from .function_call_tool_definition import FunctionCallToolDefinition +from .wolfram_alpha_tool_definition import WolframAlphaToolDefinition +from .code_interpreter_tool_definition import CodeInterpreterToolDefinition + +__all__ = ["AgentConfig", "Tool"] + +Tool: TypeAlias = Union[ + SearchToolDefinition, + WolframAlphaToolDefinition, + PhotogenToolDefinition, + CodeInterpreterToolDefinition, + FunctionCallToolDefinition, + MemoryToolDefinition, +] + + +class AgentConfig(BaseModel): + enable_session_persistence: bool + + instructions: str + + max_infer_iters: int + + model: str + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None + + sampling_params: Optional[SamplingParams] = None + + tool_choice: Optional[Literal["auto", "required"]] = None + + tool_prompt_format: Optional[Literal["json", "function_tag", "python_list"]] = None + """ + `json` -- Refers to the json format for calling tools. The json format takes the + form like { "type": "function", "function" : { "name": "function_name", + "description": "function_description", "parameters": {...} } } + + `function_tag` -- This is an example of how you could define your own user + defined format for making tool calls. The function_tag format looks like this, + (parameters) + + The detailed prompts for each of these formats are added to llama cli + """ + + tools: Optional[List[Tool]] = None diff --git a/src/llama_stack_client/types/shared/attachment.py b/src/llama_stack_client/types/shared/attachment.py index 4ad3cf7a..97a353f9 100644 --- a/src/llama_stack_client/types/shared/attachment.py +++ b/src/llama_stack_client/types/shared/attachment.py @@ -6,11 +6,11 @@ from ..._models import BaseModel from .image_media import ImageMedia -__all__ = ["Attachment", "Content", "ContentUnionMember2"] +__all__ = ["Attachment", "Content", "ContentContentArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentContentArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentContentArray]] class Attachment(BaseModel): diff --git a/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py b/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py new file mode 100644 index 00000000..ec6fb0e5 --- /dev/null +++ b/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel +from ..rest_api_execution_config import RestAPIExecutionConfig + +__all__ = ["CodeInterpreterToolDefinition"] + + +class CodeInterpreterToolDefinition(BaseModel): + enable_inline_code_execution: bool + + type: Literal["code_interpreter"] + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None + + remote_execution: Optional[RestAPIExecutionConfig] = None diff --git a/src/llama_stack_client/types/shared/completion_message.py b/src/llama_stack_client/types/shared/completion_message.py index 9b9b32f5..eb1e067c 100644 --- a/src/llama_stack_client/types/shared/completion_message.py +++ b/src/llama_stack_client/types/shared/completion_message.py @@ -7,11 +7,11 @@ from .tool_call import ToolCall from .image_media import ImageMedia -__all__ = ["CompletionMessage", "Content", "ContentUnionMember2"] +__all__ = ["CompletionMessage", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class CompletionMessage(BaseModel): diff --git a/src/llama_stack_client/types/shared/function_call_tool_definition.py b/src/llama_stack_client/types/shared/function_call_tool_definition.py new file mode 100644 index 00000000..91b73c93 --- /dev/null +++ b/src/llama_stack_client/types/shared/function_call_tool_definition.py @@ -0,0 +1,26 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel +from .tool_param_definition import ToolParamDefinition +from ..rest_api_execution_config import RestAPIExecutionConfig + +__all__ = ["FunctionCallToolDefinition"] + + +class FunctionCallToolDefinition(BaseModel): + description: str + + function_name: str + + parameters: Dict[str, ToolParamDefinition] + + type: Literal["function_call"] + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None + + remote_execution: Optional[RestAPIExecutionConfig] = None diff --git a/src/llama_stack_client/types/shared/memory_tool_definition.py b/src/llama_stack_client/types/shared/memory_tool_definition.py new file mode 100644 index 00000000..019f9373 --- /dev/null +++ b/src/llama_stack_client/types/shared/memory_tool_definition.py @@ -0,0 +1,91 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal, TypeAlias + +from ..._models import BaseModel + +__all__ = [ + "MemoryToolDefinition", + "MemoryBankConfig", + "MemoryBankConfigVector", + "MemoryBankConfigKeyValue", + "MemoryBankConfigKeyword", + "MemoryBankConfigGraph", + "QueryGeneratorConfig", + "QueryGeneratorConfigDefault", + "QueryGeneratorConfigLlm", + "QueryGeneratorConfigCustom", +] + + +class MemoryBankConfigVector(BaseModel): + bank_id: str + + type: Literal["vector"] + + +class MemoryBankConfigKeyValue(BaseModel): + bank_id: str + + keys: List[str] + + type: Literal["keyvalue"] + + +class MemoryBankConfigKeyword(BaseModel): + bank_id: str + + type: Literal["keyword"] + + +class MemoryBankConfigGraph(BaseModel): + bank_id: str + + entities: List[str] + + type: Literal["graph"] + + +MemoryBankConfig: TypeAlias = Union[ + MemoryBankConfigVector, MemoryBankConfigKeyValue, MemoryBankConfigKeyword, MemoryBankConfigGraph +] + + +class QueryGeneratorConfigDefault(BaseModel): + sep: str + + type: Literal["default"] + + +class QueryGeneratorConfigLlm(BaseModel): + model: str + + template: str + + type: Literal["llm"] + + +class QueryGeneratorConfigCustom(BaseModel): + type: Literal["custom"] + + +QueryGeneratorConfig: TypeAlias = Union[ + QueryGeneratorConfigDefault, QueryGeneratorConfigLlm, QueryGeneratorConfigCustom +] + + +class MemoryToolDefinition(BaseModel): + max_chunks: int + + max_tokens_in_context: int + + memory_bank_configs: List[MemoryBankConfig] + + query_generator_config: QueryGeneratorConfig + + type: Literal["memory"] + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None diff --git a/src/llama_stack_client/types/shared/photogen_tool_definition.py b/src/llama_stack_client/types/shared/photogen_tool_definition.py new file mode 100644 index 00000000..c1dc3915 --- /dev/null +++ b/src/llama_stack_client/types/shared/photogen_tool_definition.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel +from ..rest_api_execution_config import RestAPIExecutionConfig + +__all__ = ["PhotogenToolDefinition"] + + +class PhotogenToolDefinition(BaseModel): + type: Literal["photogen"] + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None + + remote_execution: Optional[RestAPIExecutionConfig] = None diff --git a/src/llama_stack_client/types/shared/scoring_result.py b/src/llama_stack_client/types/shared/scoring_result.py new file mode 100644 index 00000000..e19e8e6a --- /dev/null +++ b/src/llama_stack_client/types/shared/scoring_result.py @@ -0,0 +1,13 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Union + +from ..._models import BaseModel + +__all__ = ["ScoringResult"] + + +class ScoringResult(BaseModel): + aggregated_results: Dict[str, Union[bool, float, str, List[object], object, None]] + + score_rows: List[Dict[str, Union[bool, float, str, List[object], object, None]]] diff --git a/src/llama_stack_client/types/shared/search_tool_definition.py b/src/llama_stack_client/types/shared/search_tool_definition.py new file mode 100644 index 00000000..4cca48b7 --- /dev/null +++ b/src/llama_stack_client/types/shared/search_tool_definition.py @@ -0,0 +1,23 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel +from ..rest_api_execution_config import RestAPIExecutionConfig + +__all__ = ["SearchToolDefinition"] + + +class SearchToolDefinition(BaseModel): + api_key: str + + engine: Literal["bing", "brave"] + + type: Literal["brave_search"] + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None + + remote_execution: Optional[RestAPIExecutionConfig] = None diff --git a/src/llama_stack_client/types/shared/system_message.py b/src/llama_stack_client/types/shared/system_message.py index d34083fc..31f16d4e 100644 --- a/src/llama_stack_client/types/shared/system_message.py +++ b/src/llama_stack_client/types/shared/system_message.py @@ -6,11 +6,11 @@ from ..._models import BaseModel from .image_media import ImageMedia -__all__ = ["SystemMessage", "Content", "ContentUnionMember2"] +__all__ = ["SystemMessage", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class SystemMessage(BaseModel): diff --git a/src/llama_stack_client/types/shared/tool_param_definition.py b/src/llama_stack_client/types/shared/tool_param_definition.py new file mode 100644 index 00000000..1466c1f9 --- /dev/null +++ b/src/llama_stack_client/types/shared/tool_param_definition.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional + +from ..._models import BaseModel + +__all__ = ["ToolParamDefinition"] + + +class ToolParamDefinition(BaseModel): + param_type: str + + default: Union[bool, float, str, List[object], object, None] = None + + description: Optional[str] = None + + required: Optional[bool] = None diff --git a/src/llama_stack_client/types/shared/tool_response_message.py b/src/llama_stack_client/types/shared/tool_response_message.py index 4125d3e1..4203ec00 100644 --- a/src/llama_stack_client/types/shared/tool_response_message.py +++ b/src/llama_stack_client/types/shared/tool_response_message.py @@ -6,11 +6,11 @@ from ..._models import BaseModel from .image_media import ImageMedia -__all__ = ["ToolResponseMessage", "Content", "ContentUnionMember2"] +__all__ = ["ToolResponseMessage", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class ToolResponseMessage(BaseModel): diff --git a/src/llama_stack_client/types/shared/user_message.py b/src/llama_stack_client/types/shared/user_message.py index af516329..1c72bd12 100644 --- a/src/llama_stack_client/types/shared/user_message.py +++ b/src/llama_stack_client/types/shared/user_message.py @@ -6,15 +6,15 @@ from ..._models import BaseModel from .image_media import ImageMedia -__all__ = ["UserMessage", "Content", "ContentUnionMember2", "Context", "ContextUnionMember2"] +__all__ = ["UserMessage", "Content", "ContentImageMediaArray", "Context", "ContextImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] -ContextUnionMember2: TypeAlias = Union[str, ImageMedia] +ContextImageMediaArray: TypeAlias = Union[str, ImageMedia] -Context: TypeAlias = Union[str, ImageMedia, List[ContextUnionMember2]] +Context: TypeAlias = Union[str, ImageMedia, List[ContextImageMediaArray]] class UserMessage(BaseModel): diff --git a/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py b/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py new file mode 100644 index 00000000..182a0c20 --- /dev/null +++ b/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel +from ..rest_api_execution_config import RestAPIExecutionConfig + +__all__ = ["WolframAlphaToolDefinition"] + + +class WolframAlphaToolDefinition(BaseModel): + api_key: str + + type: Literal["wolfram_alpha"] + + input_shields: Optional[List[str]] = None + + output_shields: Optional[List[str]] = None + + remote_execution: Optional[RestAPIExecutionConfig] = None diff --git a/src/llama_stack_client/types/shared_params/__init__.py b/src/llama_stack_client/types/shared_params/__init__.py index 157c2d45..db829077 100644 --- a/src/llama_stack_client/types/shared_params/__init__.py +++ b/src/llama_stack_client/types/shared_params/__init__.py @@ -3,12 +3,20 @@ from .tool_call import ToolCall as ToolCall from .attachment import Attachment as Attachment from .image_media import ImageMedia as ImageMedia +from .agent_config import AgentConfig as AgentConfig from .user_message import UserMessage as UserMessage from .system_message import SystemMessage as SystemMessage from .sampling_params import SamplingParams as SamplingParams from .completion_message import CompletionMessage as CompletionMessage from .graph_memory_bank_def import GraphMemoryBankDef as GraphMemoryBankDef +from .tool_param_definition import ToolParamDefinition as ToolParamDefinition from .tool_response_message import ToolResponseMessage as ToolResponseMessage +from .memory_tool_definition import MemoryToolDefinition as MemoryToolDefinition +from .search_tool_definition import SearchToolDefinition as SearchToolDefinition from .vector_memory_bank_def import VectorMemoryBankDef as VectorMemoryBankDef from .keyword_memory_bank_def import KeywordMemoryBankDef as KeywordMemoryBankDef +from .photogen_tool_definition import PhotogenToolDefinition as PhotogenToolDefinition from .key_value_memory_bank_def import KeyValueMemoryBankDef as KeyValueMemoryBankDef +from .function_call_tool_definition import FunctionCallToolDefinition as FunctionCallToolDefinition +from .wolfram_alpha_tool_definition import WolframAlphaToolDefinition as WolframAlphaToolDefinition +from .code_interpreter_tool_definition import CodeInterpreterToolDefinition as CodeInterpreterToolDefinition diff --git a/src/llama_stack_client/types/shared_params/agent_config.py b/src/llama_stack_client/types/shared_params/agent_config.py new file mode 100644 index 00000000..f89de86b --- /dev/null +++ b/src/llama_stack_client/types/shared_params/agent_config.py @@ -0,0 +1,58 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable +from typing_extensions import Literal, Required, TypeAlias, TypedDict + +from .sampling_params import SamplingParams +from .memory_tool_definition import MemoryToolDefinition +from .search_tool_definition import SearchToolDefinition +from .photogen_tool_definition import PhotogenToolDefinition +from .function_call_tool_definition import FunctionCallToolDefinition +from .wolfram_alpha_tool_definition import WolframAlphaToolDefinition +from .code_interpreter_tool_definition import CodeInterpreterToolDefinition + +__all__ = ["AgentConfig", "Tool"] + +Tool: TypeAlias = Union[ + SearchToolDefinition, + WolframAlphaToolDefinition, + PhotogenToolDefinition, + CodeInterpreterToolDefinition, + FunctionCallToolDefinition, + MemoryToolDefinition, +] + + +class AgentConfig(TypedDict, total=False): + enable_session_persistence: Required[bool] + + instructions: Required[str] + + max_infer_iters: Required[int] + + model: Required[str] + + input_shields: List[str] + + output_shields: List[str] + + sampling_params: SamplingParams + + tool_choice: Literal["auto", "required"] + + tool_prompt_format: Literal["json", "function_tag", "python_list"] + """ + `json` -- Refers to the json format for calling tools. The json format takes the + form like { "type": "function", "function" : { "name": "function_name", + "description": "function_description", "parameters": {...} } } + + `function_tag` -- This is an example of how you could define your own user + defined format for making tool calls. The function_tag format looks like this, + (parameters) + + The detailed prompts for each of these formats are added to llama cli + """ + + tools: Iterable[Tool] diff --git a/src/llama_stack_client/types/shared_params/attachment.py b/src/llama_stack_client/types/shared_params/attachment.py index 99b90247..c2940176 100644 --- a/src/llama_stack_client/types/shared_params/attachment.py +++ b/src/llama_stack_client/types/shared_params/attachment.py @@ -7,11 +7,11 @@ from .image_media import ImageMedia -__all__ = ["Attachment", "Content", "ContentUnionMember2"] +__all__ = ["Attachment", "Content", "ContentContentArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentContentArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentContentArray]] class Attachment(TypedDict, total=False): diff --git a/src/llama_stack_client/types/shared_params/code_interpreter_tool_definition.py b/src/llama_stack_client/types/shared_params/code_interpreter_tool_definition.py new file mode 100644 index 00000000..419591bb --- /dev/null +++ b/src/llama_stack_client/types/shared_params/code_interpreter_tool_definition.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List +from typing_extensions import Literal, Required, TypedDict + +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam + +__all__ = ["CodeInterpreterToolDefinition"] + + +class CodeInterpreterToolDefinition(TypedDict, total=False): + enable_inline_code_execution: Required[bool] + + type: Required[Literal["code_interpreter"]] + + input_shields: List[str] + + output_shields: List[str] + + remote_execution: RestAPIExecutionConfigParam diff --git a/src/llama_stack_client/types/shared_params/completion_message.py b/src/llama_stack_client/types/shared_params/completion_message.py index bdc5fddd..6c5250e5 100644 --- a/src/llama_stack_client/types/shared_params/completion_message.py +++ b/src/llama_stack_client/types/shared_params/completion_message.py @@ -8,11 +8,11 @@ from .tool_call import ToolCall from .image_media import ImageMedia -__all__ = ["CompletionMessage", "Content", "ContentUnionMember2"] +__all__ = ["CompletionMessage", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class CompletionMessage(TypedDict, total=False): diff --git a/src/llama_stack_client/types/shared_params/function_call_tool_definition.py b/src/llama_stack_client/types/shared_params/function_call_tool_definition.py new file mode 100644 index 00000000..03c7aa66 --- /dev/null +++ b/src/llama_stack_client/types/shared_params/function_call_tool_definition.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict, List +from typing_extensions import Literal, Required, TypedDict + +from .tool_param_definition import ToolParamDefinition +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam + +__all__ = ["FunctionCallToolDefinition"] + + +class FunctionCallToolDefinition(TypedDict, total=False): + description: Required[str] + + function_name: Required[str] + + parameters: Required[Dict[str, ToolParamDefinition]] + + type: Required[Literal["function_call"]] + + input_shields: List[str] + + output_shields: List[str] + + remote_execution: RestAPIExecutionConfigParam diff --git a/src/llama_stack_client/types/shared_params/memory_tool_definition.py b/src/llama_stack_client/types/shared_params/memory_tool_definition.py new file mode 100644 index 00000000..442597ee --- /dev/null +++ b/src/llama_stack_client/types/shared_params/memory_tool_definition.py @@ -0,0 +1,91 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable +from typing_extensions import Literal, Required, TypeAlias, TypedDict + +__all__ = [ + "MemoryToolDefinition", + "MemoryBankConfig", + "MemoryBankConfigVector", + "MemoryBankConfigKeyValue", + "MemoryBankConfigKeyword", + "MemoryBankConfigGraph", + "QueryGeneratorConfig", + "QueryGeneratorConfigDefault", + "QueryGeneratorConfigLlm", + "QueryGeneratorConfigCustom", +] + + +class MemoryBankConfigVector(TypedDict, total=False): + bank_id: Required[str] + + type: Required[Literal["vector"]] + + +class MemoryBankConfigKeyValue(TypedDict, total=False): + bank_id: Required[str] + + keys: Required[List[str]] + + type: Required[Literal["keyvalue"]] + + +class MemoryBankConfigKeyword(TypedDict, total=False): + bank_id: Required[str] + + type: Required[Literal["keyword"]] + + +class MemoryBankConfigGraph(TypedDict, total=False): + bank_id: Required[str] + + entities: Required[List[str]] + + type: Required[Literal["graph"]] + + +MemoryBankConfig: TypeAlias = Union[ + MemoryBankConfigVector, MemoryBankConfigKeyValue, MemoryBankConfigKeyword, MemoryBankConfigGraph +] + + +class QueryGeneratorConfigDefault(TypedDict, total=False): + sep: Required[str] + + type: Required[Literal["default"]] + + +class QueryGeneratorConfigLlm(TypedDict, total=False): + model: Required[str] + + template: Required[str] + + type: Required[Literal["llm"]] + + +class QueryGeneratorConfigCustom(TypedDict, total=False): + type: Required[Literal["custom"]] + + +QueryGeneratorConfig: TypeAlias = Union[ + QueryGeneratorConfigDefault, QueryGeneratorConfigLlm, QueryGeneratorConfigCustom +] + + +class MemoryToolDefinition(TypedDict, total=False): + max_chunks: Required[int] + + max_tokens_in_context: Required[int] + + memory_bank_configs: Required[Iterable[MemoryBankConfig]] + + query_generator_config: Required[QueryGeneratorConfig] + + type: Required[Literal["memory"]] + + input_shields: List[str] + + output_shields: List[str] diff --git a/src/llama_stack_client/types/shared_params/photogen_tool_definition.py b/src/llama_stack_client/types/shared_params/photogen_tool_definition.py new file mode 100644 index 00000000..85071c38 --- /dev/null +++ b/src/llama_stack_client/types/shared_params/photogen_tool_definition.py @@ -0,0 +1,20 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List +from typing_extensions import Literal, Required, TypedDict + +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam + +__all__ = ["PhotogenToolDefinition"] + + +class PhotogenToolDefinition(TypedDict, total=False): + type: Required[Literal["photogen"]] + + input_shields: List[str] + + output_shields: List[str] + + remote_execution: RestAPIExecutionConfigParam diff --git a/src/llama_stack_client/types/shared_params/search_tool_definition.py b/src/llama_stack_client/types/shared_params/search_tool_definition.py new file mode 100644 index 00000000..80ad9b81 --- /dev/null +++ b/src/llama_stack_client/types/shared_params/search_tool_definition.py @@ -0,0 +1,24 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List +from typing_extensions import Literal, Required, TypedDict + +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam + +__all__ = ["SearchToolDefinition"] + + +class SearchToolDefinition(TypedDict, total=False): + api_key: Required[str] + + engine: Required[Literal["bing", "brave"]] + + type: Required[Literal["brave_search"]] + + input_shields: List[str] + + output_shields: List[str] + + remote_execution: RestAPIExecutionConfigParam diff --git a/src/llama_stack_client/types/shared_params/system_message.py b/src/llama_stack_client/types/shared_params/system_message.py index b33758a1..4453b4e8 100644 --- a/src/llama_stack_client/types/shared_params/system_message.py +++ b/src/llama_stack_client/types/shared_params/system_message.py @@ -7,11 +7,11 @@ from .image_media import ImageMedia -__all__ = ["SystemMessage", "Content", "ContentUnionMember2"] +__all__ = ["SystemMessage", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class SystemMessage(TypedDict, total=False): diff --git a/src/llama_stack_client/types/shared_params/tool_param_definition.py b/src/llama_stack_client/types/shared_params/tool_param_definition.py new file mode 100644 index 00000000..2d7805fe --- /dev/null +++ b/src/llama_stack_client/types/shared_params/tool_param_definition.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union, Iterable +from typing_extensions import Required, TypedDict + +__all__ = ["ToolParamDefinition"] + + +class ToolParamDefinition(TypedDict, total=False): + param_type: Required[str] + + default: Union[bool, float, str, Iterable[object], object, None] + + description: str + + required: bool diff --git a/src/llama_stack_client/types/shared_params/tool_response_message.py b/src/llama_stack_client/types/shared_params/tool_response_message.py index 3c867526..75865c49 100644 --- a/src/llama_stack_client/types/shared_params/tool_response_message.py +++ b/src/llama_stack_client/types/shared_params/tool_response_message.py @@ -7,11 +7,11 @@ from .image_media import ImageMedia -__all__ = ["ToolResponseMessage", "Content", "ContentUnionMember2"] +__all__ = ["ToolResponseMessage", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class ToolResponseMessage(TypedDict, total=False): diff --git a/src/llama_stack_client/types/shared_params/user_message.py b/src/llama_stack_client/types/shared_params/user_message.py index 233c3c53..8ccca4ac 100644 --- a/src/llama_stack_client/types/shared_params/user_message.py +++ b/src/llama_stack_client/types/shared_params/user_message.py @@ -7,15 +7,15 @@ from .image_media import ImageMedia -__all__ = ["UserMessage", "Content", "ContentUnionMember2", "Context", "ContextUnionMember2"] +__all__ = ["UserMessage", "Content", "ContentImageMediaArray", "Context", "ContextImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] -ContextUnionMember2: TypeAlias = Union[str, ImageMedia] +ContextImageMediaArray: TypeAlias = Union[str, ImageMedia] -Context: TypeAlias = Union[str, ImageMedia, List[ContextUnionMember2]] +Context: TypeAlias = Union[str, ImageMedia, List[ContextImageMediaArray]] class UserMessage(TypedDict, total=False): diff --git a/src/llama_stack_client/types/shared_params/wolfram_alpha_tool_definition.py b/src/llama_stack_client/types/shared_params/wolfram_alpha_tool_definition.py new file mode 100644 index 00000000..cb87075e --- /dev/null +++ b/src/llama_stack_client/types/shared_params/wolfram_alpha_tool_definition.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List +from typing_extensions import Literal, Required, TypedDict + +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam + +__all__ = ["WolframAlphaToolDefinition"] + + +class WolframAlphaToolDefinition(TypedDict, total=False): + api_key: Required[str] + + type: Required[Literal["wolfram_alpha"]] + + input_shields: List[str] + + output_shields: List[str] + + remote_execution: RestAPIExecutionConfigParam diff --git a/src/llama_stack_client/types/shield_def_with_provider.py b/src/llama_stack_client/types/shield_def_with_provider.py index 7adee59a..c495560c 100644 --- a/src/llama_stack_client/types/shield_def_with_provider.py +++ b/src/llama_stack_client/types/shield_def_with_provider.py @@ -1,6 +1,7 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import Dict, List, Union +from typing_extensions import Literal from .._models import BaseModel @@ -14,4 +15,6 @@ class ShieldDefWithProvider(BaseModel): provider_id: str - type: str + shield_type: str + + type: Literal["shield"] diff --git a/src/llama_stack_client/types/shield_def_with_provider_param.py b/src/llama_stack_client/types/shield_def_with_provider_param.py index 0d4df4ce..6e6bf026 100644 --- a/src/llama_stack_client/types/shield_def_with_provider_param.py +++ b/src/llama_stack_client/types/shield_def_with_provider_param.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Dict, Union, Iterable -from typing_extensions import Required, TypedDict +from typing_extensions import Literal, Required, TypedDict __all__ = ["ShieldDefWithProviderParam"] @@ -15,4 +15,6 @@ class ShieldDefWithProviderParam(TypedDict, total=False): provider_id: Required[str] - type: Required[str] + shield_type: Required[str] + + type: Required[Literal["shield"]] diff --git a/src/llama_stack_client/types/shield_retrieve_params.py b/src/llama_stack_client/types/shield_retrieve_params.py index fc03d892..1288ead0 100644 --- a/src/llama_stack_client/types/shield_retrieve_params.py +++ b/src/llama_stack_client/types/shield_retrieve_params.py @@ -10,6 +10,6 @@ class ShieldRetrieveParams(TypedDict, total=False): - shield_type: Required[str] + identifier: Required[str] x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/tool_response.py b/src/llama_stack_client/types/tool_response.py index aad08716..00bad12b 100644 --- a/src/llama_stack_client/types/tool_response.py +++ b/src/llama_stack_client/types/tool_response.py @@ -6,11 +6,11 @@ from .._models import BaseModel from .shared.image_media import ImageMedia -__all__ = ["ToolResponse", "Content", "ContentUnionMember2"] +__all__ = ["ToolResponse", "Content", "ContentImageMediaArray"] -ContentUnionMember2: TypeAlias = Union[str, ImageMedia] +ContentImageMediaArray: TypeAlias = Union[str, ImageMedia] -Content: TypeAlias = Union[str, ImageMedia, List[ContentUnionMember2]] +Content: TypeAlias = Union[str, ImageMedia, List[ContentImageMediaArray]] class ToolResponse(BaseModel): diff --git a/tests/api_resources/eval/test_jobs.py b/tests/api_resources/eval/test_jobs.py new file mode 100644 index 00000000..6e574e4d --- /dev/null +++ b/tests/api_resources/eval/test_jobs.py @@ -0,0 +1,283 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, Optional, cast + +import pytest + +from tests.utils import assert_matches_type +from llama_stack_client import LlamaStackClient, AsyncLlamaStackClient +from llama_stack_client.types.eval import ( + JobStatusResponse, + JobRetrieveResponse, +) + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestJobs: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + def test_method_retrieve(self, client: LlamaStackClient) -> None: + job = client.eval.jobs.retrieve( + job_id="job_id", + task_id="task_id", + ) + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + @parametrize + def test_method_retrieve_with_all_params(self, client: LlamaStackClient) -> None: + job = client.eval.jobs.retrieve( + job_id="job_id", + task_id="task_id", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + @parametrize + def test_raw_response_retrieve(self, client: LlamaStackClient) -> None: + response = client.eval.jobs.with_raw_response.retrieve( + job_id="job_id", + task_id="task_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + @parametrize + def test_streaming_response_retrieve(self, client: LlamaStackClient) -> None: + with client.eval.jobs.with_streaming_response.retrieve( + job_id="job_id", + task_id="task_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = response.parse() + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_cancel(self, client: LlamaStackClient) -> None: + job = client.eval.jobs.cancel( + job_id="job_id", + task_id="task_id", + ) + assert job is None + + @parametrize + def test_method_cancel_with_all_params(self, client: LlamaStackClient) -> None: + job = client.eval.jobs.cancel( + job_id="job_id", + task_id="task_id", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert job is None + + @parametrize + def test_raw_response_cancel(self, client: LlamaStackClient) -> None: + response = client.eval.jobs.with_raw_response.cancel( + job_id="job_id", + task_id="task_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert job is None + + @parametrize + def test_streaming_response_cancel(self, client: LlamaStackClient) -> None: + with client.eval.jobs.with_streaming_response.cancel( + job_id="job_id", + task_id="task_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = response.parse() + assert job is None + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_status(self, client: LlamaStackClient) -> None: + job = client.eval.jobs.status( + job_id="job_id", + task_id="task_id", + ) + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + @parametrize + def test_method_status_with_all_params(self, client: LlamaStackClient) -> None: + job = client.eval.jobs.status( + job_id="job_id", + task_id="task_id", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + @parametrize + def test_raw_response_status(self, client: LlamaStackClient) -> None: + response = client.eval.jobs.with_raw_response.status( + job_id="job_id", + task_id="task_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + @parametrize + def test_streaming_response_status(self, client: LlamaStackClient) -> None: + with client.eval.jobs.with_streaming_response.status( + job_id="job_id", + task_id="task_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = response.parse() + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + +class TestAsyncJobs: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + async def test_method_retrieve(self, async_client: AsyncLlamaStackClient) -> None: + job = await async_client.eval.jobs.retrieve( + job_id="job_id", + task_id="task_id", + ) + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + @parametrize + async def test_method_retrieve_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + job = await async_client.eval.jobs.retrieve( + job_id="job_id", + task_id="task_id", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + @parametrize + async def test_raw_response_retrieve(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval.jobs.with_raw_response.retrieve( + job_id="job_id", + task_id="task_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = await response.parse() + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + @parametrize + async def test_streaming_response_retrieve(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval.jobs.with_streaming_response.retrieve( + job_id="job_id", + task_id="task_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = await response.parse() + assert_matches_type(JobRetrieveResponse, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_cancel(self, async_client: AsyncLlamaStackClient) -> None: + job = await async_client.eval.jobs.cancel( + job_id="job_id", + task_id="task_id", + ) + assert job is None + + @parametrize + async def test_method_cancel_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + job = await async_client.eval.jobs.cancel( + job_id="job_id", + task_id="task_id", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert job is None + + @parametrize + async def test_raw_response_cancel(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval.jobs.with_raw_response.cancel( + job_id="job_id", + task_id="task_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = await response.parse() + assert job is None + + @parametrize + async def test_streaming_response_cancel(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval.jobs.with_streaming_response.cancel( + job_id="job_id", + task_id="task_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = await response.parse() + assert job is None + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_status(self, async_client: AsyncLlamaStackClient) -> None: + job = await async_client.eval.jobs.status( + job_id="job_id", + task_id="task_id", + ) + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + @parametrize + async def test_method_status_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + job = await async_client.eval.jobs.status( + job_id="job_id", + task_id="task_id", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + @parametrize + async def test_raw_response_status(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval.jobs.with_raw_response.status( + job_id="job_id", + task_id="task_id", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = await response.parse() + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + @parametrize + async def test_streaming_response_status(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval.jobs.with_streaming_response.status( + job_id="job_id", + task_id="task_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = await response.parse() + assert_matches_type(Optional[JobStatusResponse], job, path=["response"]) + + assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_datasets.py b/tests/api_resources/test_datasets.py index c85ce04b..f0f6ddd4 100644 --- a/tests/api_resources/test_datasets.py +++ b/tests/api_resources/test_datasets.py @@ -111,6 +111,7 @@ def test_method_register(self, client: LlamaStackClient) -> None: "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, ) @@ -124,6 +125,7 @@ def test_method_register_with_all_params(self, client: LlamaStackClient) -> None "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", @@ -138,6 +140,7 @@ def test_raw_response_register(self, client: LlamaStackClient) -> None: "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, ) @@ -155,6 +158,7 @@ def test_streaming_response_register(self, client: LlamaStackClient) -> None: "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, ) as response: @@ -261,6 +265,7 @@ async def test_method_register(self, async_client: AsyncLlamaStackClient) -> Non "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, ) @@ -274,6 +279,7 @@ async def test_method_register_with_all_params(self, async_client: AsyncLlamaSta "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", @@ -288,6 +294,7 @@ async def test_raw_response_register(self, async_client: AsyncLlamaStackClient) "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, ) @@ -305,6 +312,7 @@ async def test_streaming_response_register(self, async_client: AsyncLlamaStackCl "identifier": "identifier", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "dataset", "url": "https://example.com", }, ) as response: diff --git a/tests/api_resources/test_eval.py b/tests/api_resources/test_eval.py index 9a117c17..db56c35c 100644 --- a/tests/api_resources/test_eval.py +++ b/tests/api_resources/test_eval.py @@ -9,7 +9,10 @@ from tests.utils import assert_matches_type from llama_stack_client import LlamaStackClient, AsyncLlamaStackClient -from llama_stack_client.types import Job, EvalEvaluateResponse +from llama_stack_client.types import ( + EvalRunEvalResponse, + EvalEvaluateRowsResponse, +) base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -18,150 +21,176 @@ class TestEval: parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) @parametrize - def test_method_evaluate(self, client: LlamaStackClient) -> None: - eval = client.eval.evaluate( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", - }, + def test_method_evaluate_rows(self, client: LlamaStackClient) -> None: + eval = client.eval.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", + }, + task_id="task_id", ) - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) @parametrize - def test_method_evaluate_with_all_params(self, client: LlamaStackClient) -> None: - eval = client.eval.evaluate( - candidate={ - "model": "model", - "sampling_params": { - "strategy": "greedy", - "max_tokens": 0, - "repetition_penalty": 0, - "temperature": 0, - "top_k": 0, - "top_p": 0, - }, - "type": "model", - "system_message": { - "content": "string", - "role": "system", - }, - }, + def test_method_evaluate_rows_with_all_params(self, client: LlamaStackClient) -> None: + eval = client.eval.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": { + "strategy": "greedy", + "max_tokens": 0, + "repetition_penalty": 0, + "temperature": 0, + "top_k": 0, + "top_p": 0, + }, + "type": "model", + "system_message": { + "content": "string", + "role": "system", + }, + }, + "type": "benchmark", + "num_examples": 0, + }, + task_id="task_id", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) @parametrize - def test_raw_response_evaluate(self, client: LlamaStackClient) -> None: - response = client.eval.with_raw_response.evaluate( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", - }, + def test_raw_response_evaluate_rows(self, client: LlamaStackClient) -> None: + response = client.eval.with_raw_response.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", + }, + task_id="task_id", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = response.parse() - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) @parametrize - def test_streaming_response_evaluate(self, client: LlamaStackClient) -> None: - with client.eval.with_streaming_response.evaluate( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", - }, + def test_streaming_response_evaluate_rows(self, client: LlamaStackClient) -> None: + with client.eval.with_streaming_response.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", + }, + task_id="task_id", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = response.parse() - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) assert cast(Any, response.is_closed) is True @parametrize - def test_method_evaluate_batch(self, client: LlamaStackClient) -> None: - eval = client.eval.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", + def test_method_run_eval(self, client: LlamaStackClient) -> None: + eval = client.eval.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", ) - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) @parametrize - def test_method_evaluate_batch_with_all_params(self, client: LlamaStackClient) -> None: - eval = client.eval.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": { - "strategy": "greedy", - "max_tokens": 0, - "repetition_penalty": 0, - "temperature": 0, - "top_k": 0, - "top_p": 0, - }, - "type": "model", - "system_message": { - "content": "string", - "role": "system", + def test_method_run_eval_with_all_params(self, client: LlamaStackClient) -> None: + eval = client.eval.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": { + "strategy": "greedy", + "max_tokens": 0, + "repetition_penalty": 0, + "temperature": 0, + "top_k": 0, + "top_p": 0, + }, + "type": "model", + "system_message": { + "content": "string", + "role": "system", + }, }, + "type": "benchmark", + "num_examples": 0, }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) @parametrize - def test_raw_response_evaluate_batch(self, client: LlamaStackClient) -> None: - response = client.eval.with_raw_response.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", + def test_raw_response_run_eval(self, client: LlamaStackClient) -> None: + response = client.eval.with_raw_response.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = response.parse() - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) @parametrize - def test_streaming_response_evaluate_batch(self, client: LlamaStackClient) -> None: - with client.eval.with_streaming_response.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", + def test_streaming_response_run_eval(self, client: LlamaStackClient) -> None: + with client.eval.with_streaming_response.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = response.parse() - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) assert cast(Any, response.is_closed) is True @@ -170,149 +199,175 @@ class TestAsyncEval: parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) @parametrize - async def test_method_evaluate(self, async_client: AsyncLlamaStackClient) -> None: - eval = await async_client.eval.evaluate( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", - }, + async def test_method_evaluate_rows(self, async_client: AsyncLlamaStackClient) -> None: + eval = await async_client.eval.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", + }, + task_id="task_id", ) - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) @parametrize - async def test_method_evaluate_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: - eval = await async_client.eval.evaluate( - candidate={ - "model": "model", - "sampling_params": { - "strategy": "greedy", - "max_tokens": 0, - "repetition_penalty": 0, - "temperature": 0, - "top_k": 0, - "top_p": 0, - }, - "type": "model", - "system_message": { - "content": "string", - "role": "system", - }, - }, + async def test_method_evaluate_rows_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + eval = await async_client.eval.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": { + "strategy": "greedy", + "max_tokens": 0, + "repetition_penalty": 0, + "temperature": 0, + "top_k": 0, + "top_p": 0, + }, + "type": "model", + "system_message": { + "content": "string", + "role": "system", + }, + }, + "type": "benchmark", + "num_examples": 0, + }, + task_id="task_id", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) @parametrize - async def test_raw_response_evaluate(self, async_client: AsyncLlamaStackClient) -> None: - response = await async_client.eval.with_raw_response.evaluate( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", - }, + async def test_raw_response_evaluate_rows(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval.with_raw_response.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", + }, + task_id="task_id", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = await response.parse() - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) @parametrize - async def test_streaming_response_evaluate(self, async_client: AsyncLlamaStackClient) -> None: - async with async_client.eval.with_streaming_response.evaluate( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", - }, + async def test_streaming_response_evaluate_rows(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval.with_streaming_response.evaluate_rows( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], scoring_functions=["string", "string", "string"], + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", + }, + task_id="task_id", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = await response.parse() - assert_matches_type(EvalEvaluateResponse, eval, path=["response"]) + assert_matches_type(EvalEvaluateRowsResponse, eval, path=["response"]) assert cast(Any, response.is_closed) is True @parametrize - async def test_method_evaluate_batch(self, async_client: AsyncLlamaStackClient) -> None: - eval = await async_client.eval.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", + async def test_method_run_eval(self, async_client: AsyncLlamaStackClient) -> None: + eval = await async_client.eval.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", ) - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) @parametrize - async def test_method_evaluate_batch_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: - eval = await async_client.eval.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": { - "strategy": "greedy", - "max_tokens": 0, - "repetition_penalty": 0, - "temperature": 0, - "top_k": 0, - "top_p": 0, - }, - "type": "model", - "system_message": { - "content": "string", - "role": "system", + async def test_method_run_eval_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + eval = await async_client.eval.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": { + "strategy": "greedy", + "max_tokens": 0, + "repetition_penalty": 0, + "temperature": 0, + "top_k": 0, + "top_p": 0, + }, + "type": "model", + "system_message": { + "content": "string", + "role": "system", + }, }, + "type": "benchmark", + "num_examples": 0, }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) @parametrize - async def test_raw_response_evaluate_batch(self, async_client: AsyncLlamaStackClient) -> None: - response = await async_client.eval.with_raw_response.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", + async def test_raw_response_run_eval(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval.with_raw_response.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = await response.parse() - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) @parametrize - async def test_streaming_response_evaluate_batch(self, async_client: AsyncLlamaStackClient) -> None: - async with async_client.eval.with_streaming_response.evaluate_batch( - candidate={ - "model": "model", - "sampling_params": {"strategy": "greedy"}, - "type": "model", + async def test_streaming_response_run_eval(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval.with_streaming_response.run_eval( + task_config={ + "eval_candidate": { + "model": "model", + "sampling_params": {"strategy": "greedy"}, + "type": "model", + }, + "type": "benchmark", }, - dataset_id="dataset_id", - scoring_functions=["string", "string", "string"], + task_id="task_id", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" eval = await response.parse() - assert_matches_type(Job, eval, path=["response"]) + assert_matches_type(EvalRunEvalResponse, eval, path=["response"]) assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_models.py b/tests/api_resources/test_models.py index 144c967c..0fe1a29c 100644 --- a/tests/api_resources/test_models.py +++ b/tests/api_resources/test_models.py @@ -108,6 +108,7 @@ def test_method_register(self, client: LlamaStackClient) -> None: "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, ) assert model is None @@ -120,6 +121,7 @@ def test_method_register_with_all_params(self, client: LlamaStackClient) -> None "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) @@ -133,6 +135,7 @@ def test_raw_response_register(self, client: LlamaStackClient) -> None: "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, ) @@ -149,6 +152,7 @@ def test_streaming_response_register(self, client: LlamaStackClient) -> None: "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, ) as response: assert not response.is_closed @@ -254,6 +258,7 @@ async def test_method_register(self, async_client: AsyncLlamaStackClient) -> Non "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, ) assert model is None @@ -266,6 +271,7 @@ async def test_method_register_with_all_params(self, async_client: AsyncLlamaSta "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) @@ -279,6 +285,7 @@ async def test_raw_response_register(self, async_client: AsyncLlamaStackClient) "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, ) @@ -295,6 +302,7 @@ async def test_streaming_response_register(self, async_client: AsyncLlamaStackCl "llama_model": "llama_model", "metadata": {"foo": True}, "provider_id": "provider_id", + "type": "model", }, ) as response: assert not response.is_closed diff --git a/tests/api_resources/test_safety.py b/tests/api_resources/test_safety.py index d1ffba98..65ac34f2 100644 --- a/tests/api_resources/test_safety.py +++ b/tests/api_resources/test_safety.py @@ -20,6 +20,7 @@ class TestSafety: @parametrize def test_method_run_shield(self, client: LlamaStackClient) -> None: safety = client.safety.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -35,13 +36,13 @@ def test_method_run_shield(self, client: LlamaStackClient) -> None: }, ], params={"foo": True}, - shield_type="shield_type", ) assert_matches_type(RunShieldResponse, safety, path=["response"]) @parametrize def test_method_run_shield_with_all_params(self, client: LlamaStackClient) -> None: safety = client.safety.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -60,7 +61,6 @@ def test_method_run_shield_with_all_params(self, client: LlamaStackClient) -> No }, ], params={"foo": True}, - shield_type="shield_type", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) assert_matches_type(RunShieldResponse, safety, path=["response"]) @@ -68,6 +68,7 @@ def test_method_run_shield_with_all_params(self, client: LlamaStackClient) -> No @parametrize def test_raw_response_run_shield(self, client: LlamaStackClient) -> None: response = client.safety.with_raw_response.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -83,7 +84,6 @@ def test_raw_response_run_shield(self, client: LlamaStackClient) -> None: }, ], params={"foo": True}, - shield_type="shield_type", ) assert response.is_closed is True @@ -94,6 +94,7 @@ def test_raw_response_run_shield(self, client: LlamaStackClient) -> None: @parametrize def test_streaming_response_run_shield(self, client: LlamaStackClient) -> None: with client.safety.with_streaming_response.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -109,7 +110,6 @@ def test_streaming_response_run_shield(self, client: LlamaStackClient) -> None: }, ], params={"foo": True}, - shield_type="shield_type", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" @@ -126,6 +126,7 @@ class TestAsyncSafety: @parametrize async def test_method_run_shield(self, async_client: AsyncLlamaStackClient) -> None: safety = await async_client.safety.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -141,13 +142,13 @@ async def test_method_run_shield(self, async_client: AsyncLlamaStackClient) -> N }, ], params={"foo": True}, - shield_type="shield_type", ) assert_matches_type(RunShieldResponse, safety, path=["response"]) @parametrize async def test_method_run_shield_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: safety = await async_client.safety.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -166,7 +167,6 @@ async def test_method_run_shield_with_all_params(self, async_client: AsyncLlamaS }, ], params={"foo": True}, - shield_type="shield_type", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) assert_matches_type(RunShieldResponse, safety, path=["response"]) @@ -174,6 +174,7 @@ async def test_method_run_shield_with_all_params(self, async_client: AsyncLlamaS @parametrize async def test_raw_response_run_shield(self, async_client: AsyncLlamaStackClient) -> None: response = await async_client.safety.with_raw_response.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -189,7 +190,6 @@ async def test_raw_response_run_shield(self, async_client: AsyncLlamaStackClient }, ], params={"foo": True}, - shield_type="shield_type", ) assert response.is_closed is True @@ -200,6 +200,7 @@ async def test_raw_response_run_shield(self, async_client: AsyncLlamaStackClient @parametrize async def test_streaming_response_run_shield(self, async_client: AsyncLlamaStackClient) -> None: async with async_client.safety.with_streaming_response.run_shield( + identifier="identifier", messages=[ { "content": "string", @@ -215,7 +216,6 @@ async def test_streaming_response_run_shield(self, async_client: AsyncLlamaStack }, ], params={"foo": True}, - shield_type="shield_type", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" diff --git a/tests/api_resources/test_scoring.py b/tests/api_resources/test_scoring.py index d209b4db..34c6fcee 100644 --- a/tests/api_resources/test_scoring.py +++ b/tests/api_resources/test_scoring.py @@ -9,7 +9,10 @@ from tests.utils import assert_matches_type from llama_stack_client import LlamaStackClient, AsyncLlamaStackClient -from llama_stack_client.types import ScoreResponse, ScoreBatchResponse +from llama_stack_client.types import ( + ScoringScoreResponse, + ScoringScoreBatchResponse, +) base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -21,42 +24,64 @@ class TestScoring: def test_method_score(self, client: LlamaStackClient) -> None: scoring = client.scoring.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) @parametrize def test_method_score_with_all_params(self, client: LlamaStackClient) -> None: scoring = client.scoring.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + "judge_score_regexes": ["string", "string", "string"], + "prompt_template": "prompt_template", + } + }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) @parametrize def test_raw_response_score(self, client: LlamaStackClient) -> None: response = client.scoring.with_raw_response.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = response.parse() - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) @parametrize def test_streaming_response_score(self, client: LlamaStackClient) -> None: with client.scoring.with_streaming_response.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = response.parse() - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) assert cast(Any, response.is_closed) is True @@ -65,45 +90,67 @@ def test_method_score_batch(self, client: LlamaStackClient) -> None: scoring = client.scoring.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) @parametrize def test_method_score_batch_with_all_params(self, client: LlamaStackClient) -> None: scoring = client.scoring.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + "judge_score_regexes": ["string", "string", "string"], + "prompt_template": "prompt_template", + } + }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) @parametrize def test_raw_response_score_batch(self, client: LlamaStackClient) -> None: response = client.scoring.with_raw_response.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = response.parse() - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) @parametrize def test_streaming_response_score_batch(self, client: LlamaStackClient) -> None: with client.scoring.with_streaming_response.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = response.parse() - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) assert cast(Any, response.is_closed) is True @@ -115,42 +162,64 @@ class TestAsyncScoring: async def test_method_score(self, async_client: AsyncLlamaStackClient) -> None: scoring = await async_client.scoring.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) @parametrize async def test_method_score_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: scoring = await async_client.scoring.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + "judge_score_regexes": ["string", "string", "string"], + "prompt_template": "prompt_template", + } + }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) @parametrize async def test_raw_response_score(self, async_client: AsyncLlamaStackClient) -> None: response = await async_client.scoring.with_raw_response.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = await response.parse() - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) @parametrize async def test_streaming_response_score(self, async_client: AsyncLlamaStackClient) -> None: async with async_client.scoring.with_streaming_response.score( input_rows=[{"foo": True}, {"foo": True}, {"foo": True}], - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = await response.parse() - assert_matches_type(ScoreResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreResponse, scoring, path=["response"]) assert cast(Any, response.is_closed) is True @@ -159,44 +228,66 @@ async def test_method_score_batch(self, async_client: AsyncLlamaStackClient) -> scoring = await async_client.scoring.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) @parametrize async def test_method_score_batch_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: scoring = await async_client.scoring.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + "judge_score_regexes": ["string", "string", "string"], + "prompt_template": "prompt_template", + } + }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) @parametrize async def test_raw_response_score_batch(self, async_client: AsyncLlamaStackClient) -> None: response = await async_client.scoring.with_raw_response.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = await response.parse() - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) @parametrize async def test_streaming_response_score_batch(self, async_client: AsyncLlamaStackClient) -> None: async with async_client.scoring.with_streaming_response.score_batch( dataset_id="dataset_id", save_results_dataset=True, - scoring_functions=["string", "string", "string"], + scoring_functions={ + "foo": { + "judge_model": "judge_model", + "type": "llm_as_judge", + } + }, ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" scoring = await response.parse() - assert_matches_type(ScoreBatchResponse, scoring, path=["response"]) + assert_matches_type(ScoringScoreBatchResponse, scoring, path=["response"]) assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_scoring_functions.py b/tests/api_resources/test_scoring_functions.py index 9de579f8..55f7c1a7 100644 --- a/tests/api_resources/test_scoring_functions.py +++ b/tests/api_resources/test_scoring_functions.py @@ -108,22 +108,9 @@ def test_method_register(self, client: LlamaStackClient) -> None: function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, + "type": "scoring_fn", }, ) assert scoring_function is None @@ -134,31 +121,16 @@ def test_method_register_with_all_params(self, client: LlamaStackClient) -> None function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - "description": "description", - }, - { - "name": "name", - "type": {"type": "string"}, - "description": "description", - }, - { - "name": "name", - "type": {"type": "string"}, - "description": "description", - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, - "context": { + "type": "scoring_fn", + "description": "description", + "params": { "judge_model": "judge_model", - "judge_score_regex": ["string", "string", "string"], + "type": "llm_as_judge", + "judge_score_regexes": ["string", "string", "string"], "prompt_template": "prompt_template", }, - "description": "description", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) @@ -170,22 +142,9 @@ def test_raw_response_register(self, client: LlamaStackClient) -> None: function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, + "type": "scoring_fn", }, ) @@ -200,22 +159,9 @@ def test_streaming_response_register(self, client: LlamaStackClient) -> None: function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, + "type": "scoring_fn", }, ) as response: assert not response.is_closed @@ -319,22 +265,9 @@ async def test_method_register(self, async_client: AsyncLlamaStackClient) -> Non function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, + "type": "scoring_fn", }, ) assert scoring_function is None @@ -345,31 +278,16 @@ async def test_method_register_with_all_params(self, async_client: AsyncLlamaSta function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - "description": "description", - }, - { - "name": "name", - "type": {"type": "string"}, - "description": "description", - }, - { - "name": "name", - "type": {"type": "string"}, - "description": "description", - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, - "context": { + "type": "scoring_fn", + "description": "description", + "params": { "judge_model": "judge_model", - "judge_score_regex": ["string", "string", "string"], + "type": "llm_as_judge", + "judge_score_regexes": ["string", "string", "string"], "prompt_template": "prompt_template", }, - "description": "description", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) @@ -381,22 +299,9 @@ async def test_raw_response_register(self, async_client: AsyncLlamaStackClient) function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, + "type": "scoring_fn", }, ) @@ -411,22 +316,9 @@ async def test_streaming_response_register(self, async_client: AsyncLlamaStackCl function_def={ "identifier": "identifier", "metadata": {"foo": True}, - "parameters": [ - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - { - "name": "name", - "type": {"type": "string"}, - }, - ], "provider_id": "provider_id", "return_type": {"type": "string"}, + "type": "scoring_fn", }, ) as response: assert not response.is_closed diff --git a/tests/api_resources/test_shields.py b/tests/api_resources/test_shields.py index 2ef574a4..3f13a700 100644 --- a/tests/api_resources/test_shields.py +++ b/tests/api_resources/test_shields.py @@ -20,14 +20,14 @@ class TestShields: @parametrize def test_method_retrieve(self, client: LlamaStackClient) -> None: shield = client.shields.retrieve( - shield_type="shield_type", + identifier="identifier", ) assert_matches_type(Optional[ShieldDefWithProvider], shield, path=["response"]) @parametrize def test_method_retrieve_with_all_params(self, client: LlamaStackClient) -> None: shield = client.shields.retrieve( - shield_type="shield_type", + identifier="identifier", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) assert_matches_type(Optional[ShieldDefWithProvider], shield, path=["response"]) @@ -35,7 +35,7 @@ def test_method_retrieve_with_all_params(self, client: LlamaStackClient) -> None @parametrize def test_raw_response_retrieve(self, client: LlamaStackClient) -> None: response = client.shields.with_raw_response.retrieve( - shield_type="shield_type", + identifier="identifier", ) assert response.is_closed is True @@ -46,7 +46,7 @@ def test_raw_response_retrieve(self, client: LlamaStackClient) -> None: @parametrize def test_streaming_response_retrieve(self, client: LlamaStackClient) -> None: with client.shields.with_streaming_response.retrieve( - shield_type="shield_type", + identifier="identifier", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" @@ -107,7 +107,8 @@ def test_method_register(self, client: LlamaStackClient) -> None: "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, ) assert shield is None @@ -119,7 +120,8 @@ def test_method_register_with_all_params(self, client: LlamaStackClient) -> None "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) @@ -132,7 +134,8 @@ def test_raw_response_register(self, client: LlamaStackClient) -> None: "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, ) @@ -148,7 +151,8 @@ def test_streaming_response_register(self, client: LlamaStackClient) -> None: "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, ) as response: assert not response.is_closed @@ -166,14 +170,14 @@ class TestAsyncShields: @parametrize async def test_method_retrieve(self, async_client: AsyncLlamaStackClient) -> None: shield = await async_client.shields.retrieve( - shield_type="shield_type", + identifier="identifier", ) assert_matches_type(Optional[ShieldDefWithProvider], shield, path=["response"]) @parametrize async def test_method_retrieve_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: shield = await async_client.shields.retrieve( - shield_type="shield_type", + identifier="identifier", x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) assert_matches_type(Optional[ShieldDefWithProvider], shield, path=["response"]) @@ -181,7 +185,7 @@ async def test_method_retrieve_with_all_params(self, async_client: AsyncLlamaSta @parametrize async def test_raw_response_retrieve(self, async_client: AsyncLlamaStackClient) -> None: response = await async_client.shields.with_raw_response.retrieve( - shield_type="shield_type", + identifier="identifier", ) assert response.is_closed is True @@ -192,7 +196,7 @@ async def test_raw_response_retrieve(self, async_client: AsyncLlamaStackClient) @parametrize async def test_streaming_response_retrieve(self, async_client: AsyncLlamaStackClient) -> None: async with async_client.shields.with_streaming_response.retrieve( - shield_type="shield_type", + identifier="identifier", ) as response: assert not response.is_closed assert response.http_request.headers.get("X-Stainless-Lang") == "python" @@ -253,7 +257,8 @@ async def test_method_register(self, async_client: AsyncLlamaStackClient) -> Non "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, ) assert shield is None @@ -265,7 +270,8 @@ async def test_method_register_with_all_params(self, async_client: AsyncLlamaSta "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, x_llama_stack_provider_data="X-LlamaStack-ProviderData", ) @@ -278,7 +284,8 @@ async def test_raw_response_register(self, async_client: AsyncLlamaStackClient) "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, ) @@ -294,7 +301,8 @@ async def test_streaming_response_register(self, async_client: AsyncLlamaStackCl "identifier": "identifier", "params": {"foo": True}, "provider_id": "provider_id", - "type": "type", + "shield_type": "shield_type", + "type": "shield", }, ) as response: assert not response.is_closed diff --git a/tests/test_client.py b/tests/test_client.py index ae930b0e..897e0130 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -657,7 +657,7 @@ class Model(BaseModel): [3, "", 0.5], [2, "", 0.5 * 2.0], [1, "", 0.5 * 4.0], - [-1100, "", 7.8], # test large number potentially overflowing + [-1100, "", 8], # test large number potentially overflowing ], ) @mock.patch("time.time", mock.MagicMock(return_value=1696004797)) @@ -672,29 +672,21 @@ def test_parse_retry_after_header(self, remaining_retries: int, retry_after: str @mock.patch("llama_stack_client._base_client.BaseClient._calculate_retry_timeout", _low_retry_timeout) @pytest.mark.respx(base_url=base_url) def test_retrying_timeout_errors_doesnt_leak(self, respx_mock: MockRouter) -> None: - respx_mock.post("/inference/chat_completion").mock(side_effect=httpx.TimeoutException("Test timeout error")) + respx_mock.post("/models/register").mock(side_effect=httpx.TimeoutException("Test timeout error")) with pytest.raises(APITimeoutError): self.client.post( - "/inference/chat_completion", + "/models/register", body=cast( object, dict( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + } ), ), cast_to=httpx.Response, @@ -706,29 +698,21 @@ def test_retrying_timeout_errors_doesnt_leak(self, respx_mock: MockRouter) -> No @mock.patch("llama_stack_client._base_client.BaseClient._calculate_retry_timeout", _low_retry_timeout) @pytest.mark.respx(base_url=base_url) def test_retrying_status_errors_doesnt_leak(self, respx_mock: MockRouter) -> None: - respx_mock.post("/inference/chat_completion").mock(return_value=httpx.Response(500)) + respx_mock.post("/models/register").mock(return_value=httpx.Response(500)) with pytest.raises(APIStatusError): self.client.post( - "/inference/chat_completion", + "/models/register", body=cast( object, dict( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + } ), ), cast_to=httpx.Response, @@ -761,24 +745,16 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(500) return httpx.Response(200) - respx_mock.post("/inference/chat_completion").mock(side_effect=retry_handler) - - response = client.inference.with_raw_response.chat_completion( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + respx_mock.post("/models/register").mock(side_effect=retry_handler) + + response = client.models.with_raw_response.register( + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + } ) assert response.retries_taken == failures_before_success @@ -801,24 +777,16 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(500) return httpx.Response(200) - respx_mock.post("/inference/chat_completion").mock(side_effect=retry_handler) - - response = client.inference.with_raw_response.chat_completion( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + respx_mock.post("/models/register").mock(side_effect=retry_handler) + + response = client.models.with_raw_response.register( + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + }, extra_headers={"x-stainless-retry-count": Omit()}, ) @@ -841,24 +809,16 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(500) return httpx.Response(200) - respx_mock.post("/inference/chat_completion").mock(side_effect=retry_handler) - - response = client.inference.with_raw_response.chat_completion( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + respx_mock.post("/models/register").mock(side_effect=retry_handler) + + response = client.models.with_raw_response.register( + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + }, extra_headers={"x-stainless-retry-count": "42"}, ) @@ -1481,7 +1441,7 @@ class Model(BaseModel): [3, "", 0.5], [2, "", 0.5 * 2.0], [1, "", 0.5 * 4.0], - [-1100, "", 7.8], # test large number potentially overflowing + [-1100, "", 8], # test large number potentially overflowing ], ) @mock.patch("time.time", mock.MagicMock(return_value=1696004797)) @@ -1497,29 +1457,21 @@ async def test_parse_retry_after_header(self, remaining_retries: int, retry_afte @mock.patch("llama_stack_client._base_client.BaseClient._calculate_retry_timeout", _low_retry_timeout) @pytest.mark.respx(base_url=base_url) async def test_retrying_timeout_errors_doesnt_leak(self, respx_mock: MockRouter) -> None: - respx_mock.post("/inference/chat_completion").mock(side_effect=httpx.TimeoutException("Test timeout error")) + respx_mock.post("/models/register").mock(side_effect=httpx.TimeoutException("Test timeout error")) with pytest.raises(APITimeoutError): await self.client.post( - "/inference/chat_completion", + "/models/register", body=cast( object, dict( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + } ), ), cast_to=httpx.Response, @@ -1531,29 +1483,21 @@ async def test_retrying_timeout_errors_doesnt_leak(self, respx_mock: MockRouter) @mock.patch("llama_stack_client._base_client.BaseClient._calculate_retry_timeout", _low_retry_timeout) @pytest.mark.respx(base_url=base_url) async def test_retrying_status_errors_doesnt_leak(self, respx_mock: MockRouter) -> None: - respx_mock.post("/inference/chat_completion").mock(return_value=httpx.Response(500)) + respx_mock.post("/models/register").mock(return_value=httpx.Response(500)) with pytest.raises(APIStatusError): await self.client.post( - "/inference/chat_completion", + "/models/register", body=cast( object, dict( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + } ), ), cast_to=httpx.Response, @@ -1587,24 +1531,16 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(500) return httpx.Response(200) - respx_mock.post("/inference/chat_completion").mock(side_effect=retry_handler) - - response = await client.inference.with_raw_response.chat_completion( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + respx_mock.post("/models/register").mock(side_effect=retry_handler) + + response = await client.models.with_raw_response.register( + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + } ) assert response.retries_taken == failures_before_success @@ -1628,24 +1564,16 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(500) return httpx.Response(200) - respx_mock.post("/inference/chat_completion").mock(side_effect=retry_handler) - - response = await client.inference.with_raw_response.chat_completion( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + respx_mock.post("/models/register").mock(side_effect=retry_handler) + + response = await client.models.with_raw_response.register( + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + }, extra_headers={"x-stainless-retry-count": Omit()}, ) @@ -1669,24 +1597,16 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(500) return httpx.Response(200) - respx_mock.post("/inference/chat_completion").mock(side_effect=retry_handler) - - response = await client.inference.with_raw_response.chat_completion( - messages=[ - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - { - "content": "string", - "role": "user", - }, - ], - model="model", + respx_mock.post("/models/register").mock(side_effect=retry_handler) + + response = await client.models.with_raw_response.register( + model={ + "identifier": "identifier", + "llama_model": "llama_model", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "type": "model", + }, extra_headers={"x-stainless-retry-count": "42"}, ) diff --git a/tests/test_models.py b/tests/test_models.py index bf3f0e20..59458bf7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -520,19 +520,15 @@ class Model(BaseModel): assert m3.to_dict(exclude_none=True) == {} assert m3.to_dict(exclude_defaults=True) == {} - if PYDANTIC_V2: - - class Model2(BaseModel): - created_at: datetime + class Model2(BaseModel): + created_at: datetime - time_str = "2024-03-21T11:39:01.275859" - m4 = Model2.construct(created_at=time_str) - assert m4.to_dict(mode="python") == {"created_at": datetime.fromisoformat(time_str)} - assert m4.to_dict(mode="json") == {"created_at": time_str} - else: - with pytest.raises(ValueError, match="mode is only supported in Pydantic v2"): - m.to_dict(mode="json") + time_str = "2024-03-21T11:39:01.275859" + m4 = Model2.construct(created_at=time_str) + assert m4.to_dict(mode="python") == {"created_at": datetime.fromisoformat(time_str)} + assert m4.to_dict(mode="json") == {"created_at": time_str} + if not PYDANTIC_V2: with pytest.raises(ValueError, match="warnings is only supported in Pydantic v2"): m.to_dict(warnings=False) @@ -558,9 +554,6 @@ class Model(BaseModel): assert m3.model_dump(exclude_none=True) == {} if not PYDANTIC_V2: - with pytest.raises(ValueError, match="mode is only supported in Pydantic v2"): - m.model_dump(mode="json") - with pytest.raises(ValueError, match="round_trip is only supported in Pydantic v2"): m.model_dump(round_trip=True) diff --git a/tests/test_transform.py b/tests/test_transform.py index 08f75668..364c685e 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -177,17 +177,32 @@ class DateDict(TypedDict, total=False): foo: Annotated[date, PropertyInfo(format="iso8601")] +class DatetimeModel(BaseModel): + foo: datetime + + +class DateModel(BaseModel): + foo: Optional[date] + + @parametrize @pytest.mark.asyncio async def test_iso8601_format(use_async: bool) -> None: dt = datetime.fromisoformat("2023-02-23T14:16:36.337692+00:00") + tz = "Z" if PYDANTIC_V2 else "+00:00" assert await transform({"foo": dt}, DatetimeDict, use_async) == {"foo": "2023-02-23T14:16:36.337692+00:00"} # type: ignore[comparison-overlap] + assert await transform(DatetimeModel(foo=dt), Any, use_async) == {"foo": "2023-02-23T14:16:36.337692" + tz} # type: ignore[comparison-overlap] dt = dt.replace(tzinfo=None) assert await transform({"foo": dt}, DatetimeDict, use_async) == {"foo": "2023-02-23T14:16:36.337692"} # type: ignore[comparison-overlap] + assert await transform(DatetimeModel(foo=dt), Any, use_async) == {"foo": "2023-02-23T14:16:36.337692"} # type: ignore[comparison-overlap] assert await transform({"foo": None}, DateDict, use_async) == {"foo": None} # type: ignore[comparison-overlap] + assert await transform(DateModel(foo=None), Any, use_async) == {"foo": None} # type: ignore assert await transform({"foo": date.fromisoformat("2023-02-23")}, DateDict, use_async) == {"foo": "2023-02-23"} # type: ignore[comparison-overlap] + assert await transform(DateModel(foo=date.fromisoformat("2023-02-23")), DateDict, use_async) == { + "foo": "2023-02-23" + } # type: ignore[comparison-overlap] @parametrize From 2f2ff6da2d0b59fe75f97705bd3cc6b2f58c45aa Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Thu, 7 Nov 2024 23:08:25 -0800 Subject: [PATCH 2/3] evals sync & CLI --- src/llama_stack_client/_client.py | 8 + .../lib/cli/eval_tasks/__init__.py | 7 + .../lib/cli/eval_tasks/eval_tasks.py | 28 ++ .../lib/cli/eval_tasks/list.py | 43 +++ .../lib/cli/llama_stack_client.py | 8 +- src/llama_stack_client/resources/__init__.py | 14 + .../resources/eval_tasks.py | 348 ++++++++++++++++++ src/llama_stack_client/types/__init__.py | 4 + .../types/eval_task_def_with_provider.py | 22 ++ .../eval_task_def_with_provider_param.py | 22 ++ .../types/eval_task_register_params.py | 16 + .../types/eval_task_retrieve_params.py | 15 + .../code_interpreter_tool_definition.py | 5 +- .../shared/function_call_tool_definition.py | 5 +- .../types/shared/photogen_tool_definition.py | 5 +- .../types/shared/search_tool_definition.py | 5 +- .../shared/wolfram_alpha_tool_definition.py | 5 +- tests/api_resources/test_eval_tasks.py | 322 ++++++++++++++++ 18 files changed, 868 insertions(+), 14 deletions(-) create mode 100644 src/llama_stack_client/lib/cli/eval_tasks/__init__.py create mode 100644 src/llama_stack_client/lib/cli/eval_tasks/eval_tasks.py create mode 100644 src/llama_stack_client/lib/cli/eval_tasks/list.py create mode 100644 src/llama_stack_client/resources/eval_tasks.py create mode 100644 src/llama_stack_client/types/eval_task_def_with_provider.py create mode 100644 src/llama_stack_client/types/eval_task_def_with_provider_param.py create mode 100644 src/llama_stack_client/types/eval_task_register_params.py create mode 100644 src/llama_stack_client/types/eval_task_retrieve_params.py create mode 100644 tests/api_resources/test_eval_tasks.py diff --git a/src/llama_stack_client/_client.py b/src/llama_stack_client/_client.py index 41e29755..b031de50 100644 --- a/src/llama_stack_client/_client.py +++ b/src/llama_stack_client/_client.py @@ -52,6 +52,7 @@ class LlamaStackClient(SyncAPIClient): datasetio: resources.DatasetioResource scoring: resources.ScoringResource scoring_functions: resources.ScoringFunctionsResource + eval_tasks: resources.EvalTasksResource with_raw_response: LlamaStackClientWithRawResponse with_streaming_response: LlamaStackClientWithStreamedResponse @@ -121,6 +122,7 @@ def __init__( self.datasetio = resources.DatasetioResource(self) self.scoring = resources.ScoringResource(self) self.scoring_functions = resources.ScoringFunctionsResource(self) + self.eval_tasks = resources.EvalTasksResource(self) self.with_raw_response = LlamaStackClientWithRawResponse(self) self.with_streaming_response = LlamaStackClientWithStreamedResponse(self) @@ -241,6 +243,7 @@ class AsyncLlamaStackClient(AsyncAPIClient): datasetio: resources.AsyncDatasetioResource scoring: resources.AsyncScoringResource scoring_functions: resources.AsyncScoringFunctionsResource + eval_tasks: resources.AsyncEvalTasksResource with_raw_response: AsyncLlamaStackClientWithRawResponse with_streaming_response: AsyncLlamaStackClientWithStreamedResponse @@ -310,6 +313,7 @@ def __init__( self.datasetio = resources.AsyncDatasetioResource(self) self.scoring = resources.AsyncScoringResource(self) self.scoring_functions = resources.AsyncScoringFunctionsResource(self) + self.eval_tasks = resources.AsyncEvalTasksResource(self) self.with_raw_response = AsyncLlamaStackClientWithRawResponse(self) self.with_streaming_response = AsyncLlamaStackClientWithStreamedResponse(self) @@ -433,6 +437,7 @@ def __init__(self, client: LlamaStackClient) -> None: self.datasetio = resources.DatasetioResourceWithRawResponse(client.datasetio) self.scoring = resources.ScoringResourceWithRawResponse(client.scoring) self.scoring_functions = resources.ScoringFunctionsResourceWithRawResponse(client.scoring_functions) + self.eval_tasks = resources.EvalTasksResourceWithRawResponse(client.eval_tasks) class AsyncLlamaStackClientWithRawResponse: @@ -458,6 +463,7 @@ def __init__(self, client: AsyncLlamaStackClient) -> None: self.datasetio = resources.AsyncDatasetioResourceWithRawResponse(client.datasetio) self.scoring = resources.AsyncScoringResourceWithRawResponse(client.scoring) self.scoring_functions = resources.AsyncScoringFunctionsResourceWithRawResponse(client.scoring_functions) + self.eval_tasks = resources.AsyncEvalTasksResourceWithRawResponse(client.eval_tasks) class LlamaStackClientWithStreamedResponse: @@ -483,6 +489,7 @@ def __init__(self, client: LlamaStackClient) -> None: self.datasetio = resources.DatasetioResourceWithStreamingResponse(client.datasetio) self.scoring = resources.ScoringResourceWithStreamingResponse(client.scoring) self.scoring_functions = resources.ScoringFunctionsResourceWithStreamingResponse(client.scoring_functions) + self.eval_tasks = resources.EvalTasksResourceWithStreamingResponse(client.eval_tasks) class AsyncLlamaStackClientWithStreamedResponse: @@ -508,6 +515,7 @@ def __init__(self, client: AsyncLlamaStackClient) -> None: self.datasetio = resources.AsyncDatasetioResourceWithStreamingResponse(client.datasetio) self.scoring = resources.AsyncScoringResourceWithStreamingResponse(client.scoring) self.scoring_functions = resources.AsyncScoringFunctionsResourceWithStreamingResponse(client.scoring_functions) + self.eval_tasks = resources.AsyncEvalTasksResourceWithStreamingResponse(client.eval_tasks) Client = LlamaStackClient diff --git a/src/llama_stack_client/lib/cli/eval_tasks/__init__.py b/src/llama_stack_client/lib/cli/eval_tasks/__init__.py new file mode 100644 index 00000000..ae642407 --- /dev/null +++ b/src/llama_stack_client/lib/cli/eval_tasks/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .eval_tasks import EvalTasksParser # noqa diff --git a/src/llama_stack_client/lib/cli/eval_tasks/eval_tasks.py b/src/llama_stack_client/lib/cli/eval_tasks/eval_tasks.py new file mode 100644 index 00000000..f50da552 --- /dev/null +++ b/src/llama_stack_client/lib/cli/eval_tasks/eval_tasks.py @@ -0,0 +1,28 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +import argparse + +from llama_stack_client.lib.cli.eval_tasks.list import EvalTasksList + +from llama_stack_client.lib.cli.subcommand import Subcommand + + +class EvalTasksParser(Subcommand): + """List details about available eval banks type on distribution.""" + + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self.parser = subparsers.add_parser( + "eval_tasks", + prog="llama-stack-client eval_tasks", + description="Query details about available eval tasks type on distribution.", + formatter_class=argparse.RawTextHelpFormatter, + ) + + subparsers = self.parser.add_subparsers(title="eval_tasks_subcommands") + EvalTasksList.create(subparsers) diff --git a/src/llama_stack_client/lib/cli/eval_tasks/list.py b/src/llama_stack_client/lib/cli/eval_tasks/list.py new file mode 100644 index 00000000..2b60539a --- /dev/null +++ b/src/llama_stack_client/lib/cli/eval_tasks/list.py @@ -0,0 +1,43 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import argparse + +from llama_stack_client import LlamaStackClient +from llama_stack_client.lib.cli.common.utils import print_table_from_response +from llama_stack_client.lib.cli.configure import get_config +from llama_stack_client.lib.cli.subcommand import Subcommand + + +class EvalTasksList(Subcommand): + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self.parser = subparsers.add_parser( + "list", + prog="llama-stack-client eval_tasks list", + description="Show available evaluation tasks on distribution endpoint", + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self.parser.set_defaults(func=self._run_eval_tasks_list_cmd) + + def _add_arguments(self): + self.parser.add_argument( + "--endpoint", + type=str, + help="Llama Stack distribution endpoint", + ) + + def _run_eval_tasks_list_cmd(self, args: argparse.Namespace): + args.endpoint = get_config().get("endpoint") or args.endpoint + + client = LlamaStackClient( + base_url=args.endpoint, + ) + + eval_tasks_list_response = client.eval_tasks.list() + if eval_tasks_list_response: + print_table_from_response(eval_tasks_list_response) diff --git a/src/llama_stack_client/lib/cli/llama_stack_client.py b/src/llama_stack_client/lib/cli/llama_stack_client.py index 73461906..5f290324 100644 --- a/src/llama_stack_client/lib/cli/llama_stack_client.py +++ b/src/llama_stack_client/lib/cli/llama_stack_client.py @@ -9,6 +9,7 @@ from llama_stack_client.lib.cli.constants import get_config_file_path from .configure import ConfigureParser +from .eval_tasks import EvalTasksParser from .memory_banks import MemoryBanksParser from .models import ModelsParser @@ -33,6 +34,7 @@ def __init__(self): ModelsParser.create(subparsers) MemoryBanksParser.create(subparsers) ShieldsParser.create(subparsers) + EvalTasksParser.create(subparsers) ConfigureParser.create(subparsers) ProvidersParser.create(subparsers) @@ -40,13 +42,11 @@ def parse_args(self) -> argparse.Namespace: return self.parser.parse_args() def command_requires_config(self, args: argparse.Namespace) -> bool: - return not (hasattr(args.func, '__self__') and isinstance(args.func.__self__, ConfigureParser)) + return not (hasattr(args.func, "__self__") and isinstance(args.func.__self__, ConfigureParser)) def run(self, args: argparse.Namespace) -> None: if self.command_requires_config(args) and not get_config_file_path().exists(): - print( - "Config file not found. Please run 'llama-stack-client configure' to create one." - ) + print("Config file not found. Please run 'llama-stack-client configure' to create one.") return args.func(args) diff --git a/src/llama_stack_client/resources/__init__.py b/src/llama_stack_client/resources/__init__.py index 655c3975..8f3c4abb 100644 --- a/src/llama_stack_client/resources/__init__.py +++ b/src/llama_stack_client/resources/__init__.py @@ -112,6 +112,14 @@ TelemetryResourceWithStreamingResponse, AsyncTelemetryResourceWithStreamingResponse, ) +from .eval_tasks import ( + EvalTasksResource, + AsyncEvalTasksResource, + EvalTasksResourceWithRawResponse, + AsyncEvalTasksResourceWithRawResponse, + EvalTasksResourceWithStreamingResponse, + AsyncEvalTasksResourceWithStreamingResponse, +) from .memory_banks import ( MemoryBanksResource, AsyncMemoryBanksResource, @@ -268,4 +276,10 @@ "AsyncScoringFunctionsResourceWithRawResponse", "ScoringFunctionsResourceWithStreamingResponse", "AsyncScoringFunctionsResourceWithStreamingResponse", + "EvalTasksResource", + "AsyncEvalTasksResource", + "EvalTasksResourceWithRawResponse", + "AsyncEvalTasksResourceWithRawResponse", + "EvalTasksResourceWithStreamingResponse", + "AsyncEvalTasksResourceWithStreamingResponse", ] diff --git a/src/llama_stack_client/resources/eval_tasks.py b/src/llama_stack_client/resources/eval_tasks.py new file mode 100644 index 00000000..67cf3619 --- /dev/null +++ b/src/llama_stack_client/resources/eval_tasks.py @@ -0,0 +1,348 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Optional + +import httpx + +from ..types import EvalTaskDefWithProvider, eval_task_register_params, eval_task_retrieve_params +from .._types import NOT_GIVEN, Body, Query, Headers, NoneType, NotGiven +from .._utils import ( + maybe_transform, + strip_not_given, + async_maybe_transform, +) +from .._compat import cached_property +from .._resource import SyncAPIResource, AsyncAPIResource +from .._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from .._base_client import make_request_options +from ..types.eval_task_def_with_provider import EvalTaskDefWithProvider +from ..types.eval_task_def_with_provider_param import EvalTaskDefWithProviderParam + +__all__ = ["EvalTasksResource", "AsyncEvalTasksResource"] + + +class EvalTasksResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> EvalTasksResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#accessing-raw-response-data-eg-headers + """ + return EvalTasksResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> EvalTasksResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#with_streaming_response + """ + return EvalTasksResourceWithStreamingResponse(self) + + def retrieve( + self, + *, + name: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Optional[EvalTaskDefWithProvider]: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return self._get( + "/eval_tasks/get", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"name": name}, eval_task_retrieve_params.EvalTaskRetrieveParams), + ), + cast_to=EvalTaskDefWithProvider, + ) + + def list( + self, + *, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> EvalTaskDefWithProvider: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"Accept": "application/jsonl", **(extra_headers or {})} + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return self._get( + "/eval_tasks/list", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvalTaskDefWithProvider, + ) + + def register( + self, + *, + eval_task_def: EvalTaskDefWithProviderParam, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> None: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return self._post( + "/eval_tasks/register", + body=maybe_transform({"eval_task_def": eval_task_def}, eval_task_register_params.EvalTaskRegisterParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + +class AsyncEvalTasksResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncEvalTasksResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#accessing-raw-response-data-eg-headers + """ + return AsyncEvalTasksResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncEvalTasksResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/stainless-sdks/llama-stack-python#with_streaming_response + """ + return AsyncEvalTasksResourceWithStreamingResponse(self) + + async def retrieve( + self, + *, + name: str, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Optional[EvalTaskDefWithProvider]: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return await self._get( + "/eval_tasks/get", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform({"name": name}, eval_task_retrieve_params.EvalTaskRetrieveParams), + ), + cast_to=EvalTaskDefWithProvider, + ) + + async def list( + self, + *, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> EvalTaskDefWithProvider: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"Accept": "application/jsonl", **(extra_headers or {})} + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return await self._get( + "/eval_tasks/list", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=EvalTaskDefWithProvider, + ) + + async def register( + self, + *, + eval_task_def: EvalTaskDefWithProviderParam, + x_llama_stack_provider_data: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> None: + """ + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"Accept": "*/*", **(extra_headers or {})} + extra_headers = { + **strip_not_given({"X-LlamaStack-ProviderData": x_llama_stack_provider_data}), + **(extra_headers or {}), + } + return await self._post( + "/eval_tasks/register", + body=await async_maybe_transform( + {"eval_task_def": eval_task_def}, eval_task_register_params.EvalTaskRegisterParams + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=NoneType, + ) + + +class EvalTasksResourceWithRawResponse: + def __init__(self, eval_tasks: EvalTasksResource) -> None: + self._eval_tasks = eval_tasks + + self.retrieve = to_raw_response_wrapper( + eval_tasks.retrieve, + ) + self.list = to_raw_response_wrapper( + eval_tasks.list, + ) + self.register = to_raw_response_wrapper( + eval_tasks.register, + ) + + +class AsyncEvalTasksResourceWithRawResponse: + def __init__(self, eval_tasks: AsyncEvalTasksResource) -> None: + self._eval_tasks = eval_tasks + + self.retrieve = async_to_raw_response_wrapper( + eval_tasks.retrieve, + ) + self.list = async_to_raw_response_wrapper( + eval_tasks.list, + ) + self.register = async_to_raw_response_wrapper( + eval_tasks.register, + ) + + +class EvalTasksResourceWithStreamingResponse: + def __init__(self, eval_tasks: EvalTasksResource) -> None: + self._eval_tasks = eval_tasks + + self.retrieve = to_streamed_response_wrapper( + eval_tasks.retrieve, + ) + self.list = to_streamed_response_wrapper( + eval_tasks.list, + ) + self.register = to_streamed_response_wrapper( + eval_tasks.register, + ) + + +class AsyncEvalTasksResourceWithStreamingResponse: + def __init__(self, eval_tasks: AsyncEvalTasksResource) -> None: + self._eval_tasks = eval_tasks + + self.retrieve = async_to_streamed_response_wrapper( + eval_tasks.retrieve, + ) + self.list = async_to_streamed_response_wrapper( + eval_tasks.list, + ) + self.register = async_to_streamed_response_wrapper( + eval_tasks.register, + ) diff --git a/src/llama_stack_client/types/__init__.py b/src/llama_stack_client/types/__init__.py index 4c72194a..529924b8 100644 --- a/src/llama_stack_client/types/__init__.py +++ b/src/llama_stack_client/types/__init__.py @@ -66,11 +66,14 @@ from .shield_def_with_provider import ShieldDefWithProvider as ShieldDefWithProvider from .dataset_retrieve_response import DatasetRetrieveResponse as DatasetRetrieveResponse from .eval_evaluate_rows_params import EvalEvaluateRowsParams as EvalEvaluateRowsParams +from .eval_task_register_params import EvalTaskRegisterParams as EvalTaskRegisterParams +from .eval_task_retrieve_params import EvalTaskRetrieveParams as EvalTaskRetrieveParams from .memory_bank_list_response import MemoryBankListResponse as MemoryBankListResponse from .scoring_score_batch_params import ScoringScoreBatchParams as ScoringScoreBatchParams from .telemetry_get_trace_params import TelemetryGetTraceParams as TelemetryGetTraceParams from .telemetry_log_event_params import TelemetryLogEventParams as TelemetryLogEventParams from .eval_evaluate_rows_response import EvalEvaluateRowsResponse as EvalEvaluateRowsResponse +from .eval_task_def_with_provider import EvalTaskDefWithProvider as EvalTaskDefWithProvider from .inference_completion_params import InferenceCompletionParams as InferenceCompletionParams from .inference_embeddings_params import InferenceEmbeddingsParams as InferenceEmbeddingsParams from .memory_bank_register_params import MemoryBankRegisterParams as MemoryBankRegisterParams @@ -86,6 +89,7 @@ from .scoring_function_register_params import ScoringFunctionRegisterParams as ScoringFunctionRegisterParams from .scoring_function_retrieve_params import ScoringFunctionRetrieveParams as ScoringFunctionRetrieveParams from .batch_inference_completion_params import BatchInferenceCompletionParams as BatchInferenceCompletionParams +from .eval_task_def_with_provider_param import EvalTaskDefWithProviderParam as EvalTaskDefWithProviderParam from .inference_chat_completion_response import InferenceChatCompletionResponse as InferenceChatCompletionResponse from .scoring_fn_def_with_provider_param import ScoringFnDefWithProviderParam as ScoringFnDefWithProviderParam from .synthetic_data_generation_response import SyntheticDataGenerationResponse as SyntheticDataGenerationResponse diff --git a/src/llama_stack_client/types/eval_task_def_with_provider.py b/src/llama_stack_client/types/eval_task_def_with_provider.py new file mode 100644 index 00000000..3eeecadf --- /dev/null +++ b/src/llama_stack_client/types/eval_task_def_with_provider.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Union +from typing_extensions import Literal + +from .._models import BaseModel + +__all__ = ["EvalTaskDefWithProvider"] + + +class EvalTaskDefWithProvider(BaseModel): + dataset_id: str + + identifier: str + + metadata: Dict[str, Union[bool, float, str, List[object], object, None]] + + provider_id: str + + scoring_functions: List[str] + + type: Literal["eval_task"] diff --git a/src/llama_stack_client/types/eval_task_def_with_provider_param.py b/src/llama_stack_client/types/eval_task_def_with_provider_param.py new file mode 100644 index 00000000..e9b76f8a --- /dev/null +++ b/src/llama_stack_client/types/eval_task_def_with_provider_param.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict, List, Union, Iterable +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["EvalTaskDefWithProviderParam"] + + +class EvalTaskDefWithProviderParam(TypedDict, total=False): + dataset_id: Required[str] + + identifier: Required[str] + + metadata: Required[Dict[str, Union[bool, float, str, Iterable[object], object, None]]] + + provider_id: Required[str] + + scoring_functions: Required[List[str]] + + type: Required[Literal["eval_task"]] diff --git a/src/llama_stack_client/types/eval_task_register_params.py b/src/llama_stack_client/types/eval_task_register_params.py new file mode 100644 index 00000000..f73020ca --- /dev/null +++ b/src/llama_stack_client/types/eval_task_register_params.py @@ -0,0 +1,16 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, Annotated, TypedDict + +from .._utils import PropertyInfo +from .eval_task_def_with_provider_param import EvalTaskDefWithProviderParam + +__all__ = ["EvalTaskRegisterParams"] + + +class EvalTaskRegisterParams(TypedDict, total=False): + eval_task_def: Required[EvalTaskDefWithProviderParam] + + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/eval_task_retrieve_params.py b/src/llama_stack_client/types/eval_task_retrieve_params.py new file mode 100644 index 00000000..2ec99ffc --- /dev/null +++ b/src/llama_stack_client/types/eval_task_retrieve_params.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, Annotated, TypedDict + +from .._utils import PropertyInfo + +__all__ = ["EvalTaskRetrieveParams"] + + +class EvalTaskRetrieveParams(TypedDict, total=False): + name: Required[str] + + x_llama_stack_provider_data: Annotated[str, PropertyInfo(alias="X-LlamaStack-ProviderData")] diff --git a/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py b/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py index ec6fb0e5..e9031b9a 100644 --- a/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py +++ b/src/llama_stack_client/types/shared/code_interpreter_tool_definition.py @@ -1,10 +1,11 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Optional + from typing_extensions import Literal from ..._models import BaseModel -from ..rest_api_execution_config import RestAPIExecutionConfig +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam __all__ = ["CodeInterpreterToolDefinition"] @@ -18,4 +19,4 @@ class CodeInterpreterToolDefinition(BaseModel): output_shields: Optional[List[str]] = None - remote_execution: Optional[RestAPIExecutionConfig] = None + remote_execution: Optional[RestAPIExecutionConfigParam] = None diff --git a/src/llama_stack_client/types/shared/function_call_tool_definition.py b/src/llama_stack_client/types/shared/function_call_tool_definition.py index 91b73c93..6ebfa983 100644 --- a/src/llama_stack_client/types/shared/function_call_tool_definition.py +++ b/src/llama_stack_client/types/shared/function_call_tool_definition.py @@ -1,11 +1,12 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import Dict, List, Optional + from typing_extensions import Literal from ..._models import BaseModel +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam from .tool_param_definition import ToolParamDefinition -from ..rest_api_execution_config import RestAPIExecutionConfig __all__ = ["FunctionCallToolDefinition"] @@ -23,4 +24,4 @@ class FunctionCallToolDefinition(BaseModel): output_shields: Optional[List[str]] = None - remote_execution: Optional[RestAPIExecutionConfig] = None + remote_execution: Optional[RestAPIExecutionConfigParam] = None diff --git a/src/llama_stack_client/types/shared/photogen_tool_definition.py b/src/llama_stack_client/types/shared/photogen_tool_definition.py index c1dc3915..0b8b643d 100644 --- a/src/llama_stack_client/types/shared/photogen_tool_definition.py +++ b/src/llama_stack_client/types/shared/photogen_tool_definition.py @@ -1,10 +1,11 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Optional + from typing_extensions import Literal from ..._models import BaseModel -from ..rest_api_execution_config import RestAPIExecutionConfig +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam __all__ = ["PhotogenToolDefinition"] @@ -16,4 +17,4 @@ class PhotogenToolDefinition(BaseModel): output_shields: Optional[List[str]] = None - remote_execution: Optional[RestAPIExecutionConfig] = None + remote_execution: Optional[RestAPIExecutionConfigParam] = None diff --git a/src/llama_stack_client/types/shared/search_tool_definition.py b/src/llama_stack_client/types/shared/search_tool_definition.py index 4cca48b7..8186c324 100644 --- a/src/llama_stack_client/types/shared/search_tool_definition.py +++ b/src/llama_stack_client/types/shared/search_tool_definition.py @@ -1,10 +1,11 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Optional + from typing_extensions import Literal from ..._models import BaseModel -from ..rest_api_execution_config import RestAPIExecutionConfig +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam __all__ = ["SearchToolDefinition"] @@ -20,4 +21,4 @@ class SearchToolDefinition(BaseModel): output_shields: Optional[List[str]] = None - remote_execution: Optional[RestAPIExecutionConfig] = None + remote_execution: Optional[RestAPIExecutionConfigParam] = None diff --git a/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py b/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py index 182a0c20..ccee3738 100644 --- a/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py +++ b/src/llama_stack_client/types/shared/wolfram_alpha_tool_definition.py @@ -1,10 +1,11 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Optional + from typing_extensions import Literal from ..._models import BaseModel -from ..rest_api_execution_config import RestAPIExecutionConfig +from ..rest_api_execution_config_param import RestAPIExecutionConfigParam __all__ = ["WolframAlphaToolDefinition"] @@ -18,4 +19,4 @@ class WolframAlphaToolDefinition(BaseModel): output_shields: Optional[List[str]] = None - remote_execution: Optional[RestAPIExecutionConfig] = None + remote_execution: Optional[RestAPIExecutionConfigParam] = None diff --git a/tests/api_resources/test_eval_tasks.py b/tests/api_resources/test_eval_tasks.py new file mode 100644 index 00000000..cb95f060 --- /dev/null +++ b/tests/api_resources/test_eval_tasks.py @@ -0,0 +1,322 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, Optional, cast + +import pytest + +from tests.utils import assert_matches_type +from llama_stack_client import LlamaStackClient, AsyncLlamaStackClient +from llama_stack_client.types import EvalTaskDefWithProvider + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestEvalTasks: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + def test_method_retrieve(self, client: LlamaStackClient) -> None: + eval_task = client.eval_tasks.retrieve( + name="name", + ) + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + @parametrize + def test_method_retrieve_with_all_params(self, client: LlamaStackClient) -> None: + eval_task = client.eval_tasks.retrieve( + name="name", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + @parametrize + def test_raw_response_retrieve(self, client: LlamaStackClient) -> None: + response = client.eval_tasks.with_raw_response.retrieve( + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + eval_task = response.parse() + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + @parametrize + def test_streaming_response_retrieve(self, client: LlamaStackClient) -> None: + with client.eval_tasks.with_streaming_response.retrieve( + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + eval_task = response.parse() + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + def test_method_list(self, client: LlamaStackClient) -> None: + eval_task = client.eval_tasks.list() + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + def test_method_list_with_all_params(self, client: LlamaStackClient) -> None: + eval_task = client.eval_tasks.list( + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + def test_raw_response_list(self, client: LlamaStackClient) -> None: + response = client.eval_tasks.with_raw_response.list() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + eval_task = response.parse() + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + def test_streaming_response_list(self, client: LlamaStackClient) -> None: + with client.eval_tasks.with_streaming_response.list() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + eval_task = response.parse() + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_register(self, client: LlamaStackClient) -> None: + eval_task = client.eval_tasks.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + ) + assert eval_task is None + + @parametrize + def test_method_register_with_all_params(self, client: LlamaStackClient) -> None: + eval_task = client.eval_tasks.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert eval_task is None + + @parametrize + def test_raw_response_register(self, client: LlamaStackClient) -> None: + response = client.eval_tasks.with_raw_response.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + eval_task = response.parse() + assert eval_task is None + + @parametrize + def test_streaming_response_register(self, client: LlamaStackClient) -> None: + with client.eval_tasks.with_streaming_response.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + eval_task = response.parse() + assert eval_task is None + + assert cast(Any, response.is_closed) is True + + +class TestAsyncEvalTasks: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + async def test_method_retrieve(self, async_client: AsyncLlamaStackClient) -> None: + eval_task = await async_client.eval_tasks.retrieve( + name="name", + ) + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + @parametrize + async def test_method_retrieve_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + eval_task = await async_client.eval_tasks.retrieve( + name="name", + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + @parametrize + async def test_raw_response_retrieve(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval_tasks.with_raw_response.retrieve( + name="name", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + eval_task = await response.parse() + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + @parametrize + async def test_streaming_response_retrieve(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval_tasks.with_streaming_response.retrieve( + name="name", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + eval_task = await response.parse() + assert_matches_type(Optional[EvalTaskDefWithProvider], eval_task, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + async def test_method_list(self, async_client: AsyncLlamaStackClient) -> None: + eval_task = await async_client.eval_tasks.list() + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + async def test_method_list_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + eval_task = await async_client.eval_tasks.list( + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + async def test_raw_response_list(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval_tasks.with_raw_response.list() + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + eval_task = await response.parse() + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + @pytest.mark.skip( + reason="currently no good way to test endpoints with content type application/jsonl, Prism mock server will fail" + ) + @parametrize + async def test_streaming_response_list(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval_tasks.with_streaming_response.list() as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + eval_task = await response.parse() + assert_matches_type(EvalTaskDefWithProvider, eval_task, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_register(self, async_client: AsyncLlamaStackClient) -> None: + eval_task = await async_client.eval_tasks.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + ) + assert eval_task is None + + @parametrize + async def test_method_register_with_all_params(self, async_client: AsyncLlamaStackClient) -> None: + eval_task = await async_client.eval_tasks.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + x_llama_stack_provider_data="X-LlamaStack-ProviderData", + ) + assert eval_task is None + + @parametrize + async def test_raw_response_register(self, async_client: AsyncLlamaStackClient) -> None: + response = await async_client.eval_tasks.with_raw_response.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + eval_task = await response.parse() + assert eval_task is None + + @parametrize + async def test_streaming_response_register(self, async_client: AsyncLlamaStackClient) -> None: + async with async_client.eval_tasks.with_streaming_response.register( + eval_task_def={ + "dataset_id": "dataset_id", + "identifier": "identifier", + "metadata": {"foo": True}, + "provider_id": "provider_id", + "scoring_functions": ["string", "string", "string"], + "type": "eval_task", + }, + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + eval_task = await response.parse() + assert eval_task is None + + assert cast(Any, response.is_closed) is True From 781dc60e72339c776b3740da84e24a1519b962de Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Thu, 7 Nov 2024 23:15:07 -0800 Subject: [PATCH 3/3] lists CLI --- .../lib/cli/datasets/__init__.py | 9 ++++ .../lib/cli/datasets/datasets.py | 27 ++++++++++ .../lib/cli/datasets/list.py | 45 +++++++++++++++++ .../lib/cli/llama_stack_client.py | 4 ++ .../lib/cli/scoring_functions/__init__.py | 9 ++++ .../lib/cli/scoring_functions/list.py | 50 +++++++++++++++++++ .../scoring_functions/scoring_functions.py | 27 ++++++++++ 7 files changed, 171 insertions(+) create mode 100644 src/llama_stack_client/lib/cli/datasets/__init__.py create mode 100644 src/llama_stack_client/lib/cli/datasets/datasets.py create mode 100644 src/llama_stack_client/lib/cli/datasets/list.py create mode 100644 src/llama_stack_client/lib/cli/scoring_functions/__init__.py create mode 100644 src/llama_stack_client/lib/cli/scoring_functions/list.py create mode 100644 src/llama_stack_client/lib/cli/scoring_functions/scoring_functions.py diff --git a/src/llama_stack_client/lib/cli/datasets/__init__.py b/src/llama_stack_client/lib/cli/datasets/__init__.py new file mode 100644 index 00000000..049d72ee --- /dev/null +++ b/src/llama_stack_client/lib/cli/datasets/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .datasets import DatasetsParser + +__all__ = ["DatasetsParser"] diff --git a/src/llama_stack_client/lib/cli/datasets/datasets.py b/src/llama_stack_client/lib/cli/datasets/datasets.py new file mode 100644 index 00000000..e53743c7 --- /dev/null +++ b/src/llama_stack_client/lib/cli/datasets/datasets.py @@ -0,0 +1,27 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import argparse + +from llama_stack_client.lib.cli.subcommand import Subcommand +from .list import DatasetsList + + +class DatasetsParser(Subcommand): + """Parser for datasets commands""" + + @classmethod + def create(cls, subparsers: argparse._SubParsersAction): + parser = subparsers.add_parser( + "datasets", + help="Manage datasets", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.set_defaults(func=lambda _: parser.print_help()) + + # Create subcommands + datasets_subparsers = parser.add_subparsers(title="subcommands") + DatasetsList(datasets_subparsers) diff --git a/src/llama_stack_client/lib/cli/datasets/list.py b/src/llama_stack_client/lib/cli/datasets/list.py new file mode 100644 index 00000000..923e5548 --- /dev/null +++ b/src/llama_stack_client/lib/cli/datasets/list.py @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import argparse + +from llama_stack_client import LlamaStackClient +from llama_stack_client.lib.cli.common.utils import print_table_from_response +from llama_stack_client.lib.cli.configure import get_config +from llama_stack_client.lib.cli.subcommand import Subcommand + + +class DatasetsList(Subcommand): + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self.parser = subparsers.add_parser( + "list", + prog="llama-stack-client datasets list", + description="Show available datasets on distribution endpoint", + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self.parser.set_defaults(func=self._run_datasets_list_cmd) + + def _add_arguments(self): + self.parser.add_argument( + "--endpoint", + type=str, + help="Llama Stack distribution endpoint", + ) + + def _run_datasets_list_cmd(self, args: argparse.Namespace): + args.endpoint = get_config().get("endpoint") or args.endpoint + + client = LlamaStackClient( + base_url=args.endpoint, + ) + + headers = ["identifier", "provider_id", "metadata", "type"] + + datasets_list_response = client.datasets.list() + if datasets_list_response: + print_table_from_response(datasets_list_response, headers) diff --git a/src/llama_stack_client/lib/cli/llama_stack_client.py b/src/llama_stack_client/lib/cli/llama_stack_client.py index 5f290324..3631aa25 100644 --- a/src/llama_stack_client/lib/cli/llama_stack_client.py +++ b/src/llama_stack_client/lib/cli/llama_stack_client.py @@ -9,11 +9,13 @@ from llama_stack_client.lib.cli.constants import get_config_file_path from .configure import ConfigureParser +from .datasets import DatasetsParser from .eval_tasks import EvalTasksParser from .memory_banks import MemoryBanksParser from .models import ModelsParser from .providers import ProvidersParser +from .scoring_functions import ScoringFunctionsParser from .shields import ShieldsParser @@ -37,6 +39,8 @@ def __init__(self): EvalTasksParser.create(subparsers) ConfigureParser.create(subparsers) ProvidersParser.create(subparsers) + DatasetsParser.create(subparsers) + ScoringFunctionsParser.create(subparsers) def parse_args(self) -> argparse.Namespace: return self.parser.parse_args() diff --git a/src/llama_stack_client/lib/cli/scoring_functions/__init__.py b/src/llama_stack_client/lib/cli/scoring_functions/__init__.py new file mode 100644 index 00000000..98dea0bf --- /dev/null +++ b/src/llama_stack_client/lib/cli/scoring_functions/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .scoring_functions import ScoringFunctionsParser + +__all__ = ["ScoringFunctionsParser"] diff --git a/src/llama_stack_client/lib/cli/scoring_functions/list.py b/src/llama_stack_client/lib/cli/scoring_functions/list.py new file mode 100644 index 00000000..2de0acfb --- /dev/null +++ b/src/llama_stack_client/lib/cli/scoring_functions/list.py @@ -0,0 +1,50 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import argparse + +from llama_stack_client import LlamaStackClient +from llama_stack_client.lib.cli.common.utils import print_table_from_response +from llama_stack_client.lib.cli.configure import get_config +from llama_stack_client.lib.cli.subcommand import Subcommand + + +class ScoringFunctionsList(Subcommand): + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self.parser = subparsers.add_parser( + "list", + prog="llama-stack-client scoring_functions list", + description="Show available scoring functions on distribution endpoint", + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self.parser.set_defaults(func=self._run_scoring_functions_list_cmd) + + def _add_arguments(self): + self.parser.add_argument( + "--endpoint", + type=str, + help="Llama Stack distribution endpoint", + ) + + def _run_scoring_functions_list_cmd(self, args: argparse.Namespace): + args.endpoint = get_config().get("endpoint") or args.endpoint + + client = LlamaStackClient( + base_url=args.endpoint, + ) + + headers = [ + "identifier", + "provider_id", + "description", + "type", + ] + + scoring_functions_list_response = client.scoring_functions.list() + if scoring_functions_list_response: + print_table_from_response(scoring_functions_list_response, headers) diff --git a/src/llama_stack_client/lib/cli/scoring_functions/scoring_functions.py b/src/llama_stack_client/lib/cli/scoring_functions/scoring_functions.py new file mode 100644 index 00000000..80593420 --- /dev/null +++ b/src/llama_stack_client/lib/cli/scoring_functions/scoring_functions.py @@ -0,0 +1,27 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import argparse + +from llama_stack_client.lib.cli.subcommand import Subcommand +from .list import ScoringFunctionsList + + +class ScoringFunctionsParser(Subcommand): + """Parser for scoring functions commands""" + + @classmethod + def create(cls, subparsers: argparse._SubParsersAction): + parser = subparsers.add_parser( + "scoring_functions", + help="Manage scoring functions", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.set_defaults(func=lambda _: parser.print_help()) + + # Create subcommands + scoring_functions_subparsers = parser.add_subparsers(title="subcommands") + ScoringFunctionsList(scoring_functions_subparsers)