Sync evals/scoring/dataset api

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llama_stack_client"
-version = "0.0.47"
+version = "0.0.48"
 description = "The official Python library for the llama-stack-client API"
 dynamic = ["readme"]
 license = "Apache-2.0"

src/llama_stack/lib/.keep

@@ -0,0 +1,4 @@
+File generated from our OpenAPI spec by Stainless.
+
+This directory can be used to store custom files to expand the SDK.
+It is ignored by Stainless code generation and its content (other than this keep file) won't be touched.

src/llama_stack_client/_base_client.py

@@ -1575,7 +1575,7 @@ async def _request(
         except Exception as err:
             log.debug("Encountered Exception", exc_info=True)
 
-            if retries_taken > 0:
+            if remaining_retries > 0:
                 return await self._retry_request(
                     input_options,
                     cast_to,

src/llama_stack_client/_client.py

@@ -48,22 +48,23 @@
 class LlamaStackClient(SyncAPIClient):
     agents: resources.AgentsResource
     batch_inferences: resources.BatchInferencesResource
-    datasets: resources.DatasetsResource
-    evaluate: resources.EvaluateResource
-    evaluations: resources.EvaluationsResource
     inspect: resources.InspectResource
     inference: resources.InferenceResource
     memory: resources.MemoryResource
     memory_banks: resources.MemoryBanksResource
+    datasets: resources.DatasetsResource
     models: resources.ModelsResource
     post_training: resources.PostTrainingResource
     providers: resources.ProvidersResource
-    reward_scoring: resources.RewardScoringResource
     routes: resources.RoutesResource
     safety: resources.SafetyResource
     shields: resources.ShieldsResource
     synthetic_data_generation: resources.SyntheticDataGenerationResource
     telemetry: resources.TelemetryResource
+    datasetio: resources.DatasetioResource
+    scoring: resources.ScoringResource
+    scoring_functions: resources.ScoringFunctionsResource
+    eval: resources.EvalResource
     with_raw_response: LlamaStackClientWithRawResponse
     with_streaming_response: LlamaStackClientWithStreamedResponse
 
@@ -110,22 +111,23 @@ def __init__(
 
         self.agents = resources.AgentsResource(self)
         self.batch_inferences = resources.BatchInferencesResource(self)
-        self.datasets = resources.DatasetsResource(self)
-        self.evaluate = resources.EvaluateResource(self)
-        self.evaluations = resources.EvaluationsResource(self)
         self.inspect = resources.InspectResource(self)
         self.inference = resources.InferenceResource(self)
         self.memory = resources.MemoryResource(self)
         self.memory_banks = resources.MemoryBanksResource(self)
+        self.datasets = resources.DatasetsResource(self)
         self.models = resources.ModelsResource(self)
         self.post_training = resources.PostTrainingResource(self)
         self.providers = resources.ProvidersResource(self)
-        self.reward_scoring = resources.RewardScoringResource(self)
         self.routes = resources.RoutesResource(self)
         self.safety = resources.SafetyResource(self)
         self.shields = resources.ShieldsResource(self)
         self.synthetic_data_generation = resources.SyntheticDataGenerationResource(self)
         self.telemetry = resources.TelemetryResource(self)
+        self.datasetio = resources.DatasetioResource(self)
+        self.scoring = resources.ScoringResource(self)
+        self.scoring_functions = resources.ScoringFunctionsResource(self)
+        self.eval = resources.EvalResource(self)
         self.with_raw_response = LlamaStackClientWithRawResponse(self)
         self.with_streaming_response = LlamaStackClientWithStreamedResponse(self)
 
@@ -229,22 +231,23 @@ def _make_status_error(
 class AsyncLlamaStackClient(AsyncAPIClient):
     agents: resources.AsyncAgentsResource
     batch_inferences: resources.AsyncBatchInferencesResource
-    datasets: resources.AsyncDatasetsResource
-    evaluate: resources.AsyncEvaluateResource
-    evaluations: resources.AsyncEvaluationsResource
     inspect: resources.AsyncInspectResource
     inference: resources.AsyncInferenceResource
     memory: resources.AsyncMemoryResource
     memory_banks: resources.AsyncMemoryBanksResource
+    datasets: resources.AsyncDatasetsResource
     models: resources.AsyncModelsResource
     post_training: resources.AsyncPostTrainingResource
     providers: resources.AsyncProvidersResource
-    reward_scoring: resources.AsyncRewardScoringResource
     routes: resources.AsyncRoutesResource
     safety: resources.AsyncSafetyResource
     shields: resources.AsyncShieldsResource
     synthetic_data_generation: resources.AsyncSyntheticDataGenerationResource
     telemetry: resources.AsyncTelemetryResource
+    datasetio: resources.AsyncDatasetioResource
+    scoring: resources.AsyncScoringResource
+    scoring_functions: resources.AsyncScoringFunctionsResource
+    eval: resources.AsyncEvalResource
     with_raw_response: AsyncLlamaStackClientWithRawResponse
     with_streaming_response: AsyncLlamaStackClientWithStreamedResponse
 
@@ -291,22 +294,23 @@ def __init__(
 
         self.agents = resources.AsyncAgentsResource(self)
         self.batch_inferences = resources.AsyncBatchInferencesResource(self)
-        self.datasets = resources.AsyncDatasetsResource(self)
-        self.evaluate = resources.AsyncEvaluateResource(self)
-        self.evaluations = resources.AsyncEvaluationsResource(self)
         self.inspect = resources.AsyncInspectResource(self)
         self.inference = resources.AsyncInferenceResource(self)
         self.memory = resources.AsyncMemoryResource(self)
         self.memory_banks = resources.AsyncMemoryBanksResource(self)
+        self.datasets = resources.AsyncDatasetsResource(self)
         self.models = resources.AsyncModelsResource(self)
         self.post_training = resources.AsyncPostTrainingResource(self)
         self.providers = resources.AsyncProvidersResource(self)
-        self.reward_scoring = resources.AsyncRewardScoringResource(self)
         self.routes = resources.AsyncRoutesResource(self)
         self.safety = resources.AsyncSafetyResource(self)
         self.shields = resources.AsyncShieldsResource(self)
         self.synthetic_data_generation = resources.AsyncSyntheticDataGenerationResource(self)
         self.telemetry = resources.AsyncTelemetryResource(self)
+        self.datasetio = resources.AsyncDatasetioResource(self)
+        self.scoring = resources.AsyncScoringResource(self)
+        self.scoring_functions = resources.AsyncScoringFunctionsResource(self)
+        self.eval = resources.AsyncEvalResource(self)
         self.with_raw_response = AsyncLlamaStackClientWithRawResponse(self)
         self.with_streaming_response = AsyncLlamaStackClientWithStreamedResponse(self)
 
@@ -411,96 +415,100 @@ class LlamaStackClientWithRawResponse:
     def __init__(self, client: LlamaStackClient) -> None:
         self.agents = resources.AgentsResourceWithRawResponse(client.agents)
         self.batch_inferences = resources.BatchInferencesResourceWithRawResponse(client.batch_inferences)
-        self.datasets = resources.DatasetsResourceWithRawResponse(client.datasets)
-        self.evaluate = resources.EvaluateResourceWithRawResponse(client.evaluate)
-        self.evaluations = resources.EvaluationsResourceWithRawResponse(client.evaluations)
         self.inspect = resources.InspectResourceWithRawResponse(client.inspect)
         self.inference = resources.InferenceResourceWithRawResponse(client.inference)
         self.memory = resources.MemoryResourceWithRawResponse(client.memory)
         self.memory_banks = resources.MemoryBanksResourceWithRawResponse(client.memory_banks)
+        self.datasets = resources.DatasetsResourceWithRawResponse(client.datasets)
         self.models = resources.ModelsResourceWithRawResponse(client.models)
         self.post_training = resources.PostTrainingResourceWithRawResponse(client.post_training)
         self.providers = resources.ProvidersResourceWithRawResponse(client.providers)
-        self.reward_scoring = resources.RewardScoringResourceWithRawResponse(client.reward_scoring)
         self.routes = resources.RoutesResourceWithRawResponse(client.routes)
         self.safety = resources.SafetyResourceWithRawResponse(client.safety)
         self.shields = resources.ShieldsResourceWithRawResponse(client.shields)
         self.synthetic_data_generation = resources.SyntheticDataGenerationResourceWithRawResponse(
             client.synthetic_data_generation
         )
         self.telemetry = resources.TelemetryResourceWithRawResponse(client.telemetry)
+        self.datasetio = resources.DatasetioResourceWithRawResponse(client.datasetio)
+        self.scoring = resources.ScoringResourceWithRawResponse(client.scoring)
+        self.scoring_functions = resources.ScoringFunctionsResourceWithRawResponse(client.scoring_functions)
+        self.eval = resources.EvalResourceWithRawResponse(client.eval)
 
 
 class AsyncLlamaStackClientWithRawResponse:
     def __init__(self, client: AsyncLlamaStackClient) -> None:
         self.agents = resources.AsyncAgentsResourceWithRawResponse(client.agents)
         self.batch_inferences = resources.AsyncBatchInferencesResourceWithRawResponse(client.batch_inferences)
-        self.datasets = resources.AsyncDatasetsResourceWithRawResponse(client.datasets)
-        self.evaluate = resources.AsyncEvaluateResourceWithRawResponse(client.evaluate)
-        self.evaluations = resources.AsyncEvaluationsResourceWithRawResponse(client.evaluations)
         self.inspect = resources.AsyncInspectResourceWithRawResponse(client.inspect)
         self.inference = resources.AsyncInferenceResourceWithRawResponse(client.inference)
         self.memory = resources.AsyncMemoryResourceWithRawResponse(client.memory)
         self.memory_banks = resources.AsyncMemoryBanksResourceWithRawResponse(client.memory_banks)
+        self.datasets = resources.AsyncDatasetsResourceWithRawResponse(client.datasets)
         self.models = resources.AsyncModelsResourceWithRawResponse(client.models)
         self.post_training = resources.AsyncPostTrainingResourceWithRawResponse(client.post_training)
         self.providers = resources.AsyncProvidersResourceWithRawResponse(client.providers)
-        self.reward_scoring = resources.AsyncRewardScoringResourceWithRawResponse(client.reward_scoring)
         self.routes = resources.AsyncRoutesResourceWithRawResponse(client.routes)
         self.safety = resources.AsyncSafetyResourceWithRawResponse(client.safety)
         self.shields = resources.AsyncShieldsResourceWithRawResponse(client.shields)
         self.synthetic_data_generation = resources.AsyncSyntheticDataGenerationResourceWithRawResponse(
             client.synthetic_data_generation
         )
         self.telemetry = resources.AsyncTelemetryResourceWithRawResponse(client.telemetry)
+        self.datasetio = resources.AsyncDatasetioResourceWithRawResponse(client.datasetio)
+        self.scoring = resources.AsyncScoringResourceWithRawResponse(client.scoring)
+        self.scoring_functions = resources.AsyncScoringFunctionsResourceWithRawResponse(client.scoring_functions)
+        self.eval = resources.AsyncEvalResourceWithRawResponse(client.eval)
 
 
 class LlamaStackClientWithStreamedResponse:
     def __init__(self, client: LlamaStackClient) -> None:
         self.agents = resources.AgentsResourceWithStreamingResponse(client.agents)
         self.batch_inferences = resources.BatchInferencesResourceWithStreamingResponse(client.batch_inferences)
-        self.datasets = resources.DatasetsResourceWithStreamingResponse(client.datasets)
-        self.evaluate = resources.EvaluateResourceWithStreamingResponse(client.evaluate)
-        self.evaluations = resources.EvaluationsResourceWithStreamingResponse(client.evaluations)
         self.inspect = resources.InspectResourceWithStreamingResponse(client.inspect)
         self.inference = resources.InferenceResourceWithStreamingResponse(client.inference)
         self.memory = resources.MemoryResourceWithStreamingResponse(client.memory)
         self.memory_banks = resources.MemoryBanksResourceWithStreamingResponse(client.memory_banks)
+        self.datasets = resources.DatasetsResourceWithStreamingResponse(client.datasets)
         self.models = resources.ModelsResourceWithStreamingResponse(client.models)
         self.post_training = resources.PostTrainingResourceWithStreamingResponse(client.post_training)
         self.providers = resources.ProvidersResourceWithStreamingResponse(client.providers)
-        self.reward_scoring = resources.RewardScoringResourceWithStreamingResponse(client.reward_scoring)
         self.routes = resources.RoutesResourceWithStreamingResponse(client.routes)
         self.safety = resources.SafetyResourceWithStreamingResponse(client.safety)
         self.shields = resources.ShieldsResourceWithStreamingResponse(client.shields)
         self.synthetic_data_generation = resources.SyntheticDataGenerationResourceWithStreamingResponse(
             client.synthetic_data_generation
         )
         self.telemetry = resources.TelemetryResourceWithStreamingResponse(client.telemetry)
+        self.datasetio = resources.DatasetioResourceWithStreamingResponse(client.datasetio)
+        self.scoring = resources.ScoringResourceWithStreamingResponse(client.scoring)
+        self.scoring_functions = resources.ScoringFunctionsResourceWithStreamingResponse(client.scoring_functions)
+        self.eval = resources.EvalResourceWithStreamingResponse(client.eval)
 
 
 class AsyncLlamaStackClientWithStreamedResponse:
     def __init__(self, client: AsyncLlamaStackClient) -> None:
         self.agents = resources.AsyncAgentsResourceWithStreamingResponse(client.agents)
         self.batch_inferences = resources.AsyncBatchInferencesResourceWithStreamingResponse(client.batch_inferences)
-        self.datasets = resources.AsyncDatasetsResourceWithStreamingResponse(client.datasets)
-        self.evaluate = resources.AsyncEvaluateResourceWithStreamingResponse(client.evaluate)
-        self.evaluations = resources.AsyncEvaluationsResourceWithStreamingResponse(client.evaluations)
         self.inspect = resources.AsyncInspectResourceWithStreamingResponse(client.inspect)
         self.inference = resources.AsyncInferenceResourceWithStreamingResponse(client.inference)
         self.memory = resources.AsyncMemoryResourceWithStreamingResponse(client.memory)
         self.memory_banks = resources.AsyncMemoryBanksResourceWithStreamingResponse(client.memory_banks)
+        self.datasets = resources.AsyncDatasetsResourceWithStreamingResponse(client.datasets)
         self.models = resources.AsyncModelsResourceWithStreamingResponse(client.models)
         self.post_training = resources.AsyncPostTrainingResourceWithStreamingResponse(client.post_training)
         self.providers = resources.AsyncProvidersResourceWithStreamingResponse(client.providers)
-        self.reward_scoring = resources.AsyncRewardScoringResourceWithStreamingResponse(client.reward_scoring)
         self.routes = resources.AsyncRoutesResourceWithStreamingResponse(client.routes)
         self.safety = resources.AsyncSafetyResourceWithStreamingResponse(client.safety)
         self.shields = resources.AsyncShieldsResourceWithStreamingResponse(client.shields)
         self.synthetic_data_generation = resources.AsyncSyntheticDataGenerationResourceWithStreamingResponse(
             client.synthetic_data_generation
         )
         self.telemetry = resources.AsyncTelemetryResourceWithStreamingResponse(client.telemetry)
+        self.datasetio = resources.AsyncDatasetioResourceWithStreamingResponse(client.datasetio)
+        self.scoring = resources.AsyncScoringResourceWithStreamingResponse(client.scoring)
+        self.scoring_functions = resources.AsyncScoringFunctionsResourceWithStreamingResponse(client.scoring_functions)
+        self.eval = resources.AsyncEvalResourceWithStreamingResponse(client.eval)
 
 
 Client = LlamaStackClient

src/llama_stack_client/_compat.py

@@ -133,7 +133,7 @@ def model_json(model: pydantic.BaseModel, *, indent: int | None = None) -> str:
 def model_dump(
     model: pydantic.BaseModel,
     *,
-    exclude: IncEx = None,
+    exclude: IncEx | None = None,
     exclude_unset: bool = False,
     exclude_defaults: bool = False,
     warnings: bool = True,

src/llama_stack_client/_models.py

@@ -176,7 +176,7 @@ def __str__(self) -> str:
     # Based on https://github.com/samuelcolvin/pydantic/issues/1168#issuecomment-817742836.
     @classmethod
     @override
-    def construct(
+    def construct(  # pyright: ignore[reportIncompatibleMethodOverride]
         cls: Type[ModelT],
         _fields_set: set[str] | None = None,
         **values: object,
@@ -248,8 +248,8 @@ def model_dump(
             self,
             *,
             mode: Literal["json", "python"] | str = "python",
-            include: IncEx = None,
-            exclude: IncEx = None,
+            include: IncEx | None = None,
+            exclude: IncEx | None = None,
             by_alias: bool = False,
             exclude_unset: bool = False,
             exclude_defaults: bool = False,
@@ -303,8 +303,8 @@ def model_dump_json(
             self,
             *,
             indent: int | None = None,
-            include: IncEx = None,
-            exclude: IncEx = None,
+            include: IncEx | None = None,
+            exclude: IncEx | None = None,
             by_alias: bool = False,
             exclude_unset: bool = False,
             exclude_defaults: bool = False,

src/llama_stack_client/_types.py

@@ -16,7 +16,7 @@
     Optional,
     Sequence,
 )
-from typing_extensions import Literal, Protocol, TypeAlias, TypedDict, override, runtime_checkable
+from typing_extensions import Set, Literal, Protocol, TypeAlias, TypedDict, override, runtime_checkable
 
 import httpx
 import pydantic
@@ -193,7 +193,9 @@ def get(self, __key: str) -> str | None: ...
 
 # Note: copied from Pydantic
 # https://github.com/pydantic/pydantic/blob/32ea570bf96e84234d2992e1ddf40ab8a565925a/pydantic/main.py#L49
-IncEx: TypeAlias = "set[int] | set[str] | dict[int, Any] | dict[str, Any] | None"
+IncEx: TypeAlias = Union[
+    Set[int], Set[str], Mapping[int, Union["IncEx", Literal[True]]], Mapping[str, Union["IncEx", Literal[True]]]
+]
 
 PostParser = Callable[[Any], Any]