add inference v3 support - testing

V2arK · V2arK · commit 6367b1cab1f3 · 2025-09-02T18:19:57.000-04:00
diff --git a/centml/cli/cluster.py b/centml/cli/cluster.py
@@ -13,6 +13,7 @@
     DeploymentType.COMPUTE: "compute",
     DeploymentType.COMPILATION: "compilation",
     DeploymentType.INFERENCE_V2: "inference",
+    DeploymentType.INFERENCE_V3: "inference",
     DeploymentType.COMPUTE_V2: "compute",
     # For user, they are all cserve.
     DeploymentType.CSERVE: "cserve",
@@ -22,7 +23,7 @@
 }
 # use latest type to for user requests
 depl_name_to_type_map = {
-    "inference": DeploymentType.INFERENCE_V2,
+    "inference": DeploymentType.INFERENCE_V3,
     "cserve": DeploymentType.CSERVE_V3,
     "compute": DeploymentType.COMPUTE_V2,
     "rag": DeploymentType.RAG,
@@ -140,8 +141,8 @@ def get(type, id):
     with get_centml_client() as cclient:
         depl_type = depl_name_to_type_map[type]
 
-        if depl_type == DeploymentType.INFERENCE_V2:
-            deployment = cclient.get_inference(id)
+        if depl_type in [DeploymentType.INFERENCE_V2, DeploymentType.INFERENCE_V3]:
+            deployment = cclient.get_inference(id)  # handles both V2 and V3
         elif depl_type == DeploymentType.COMPUTE_V2:
             deployment = cclient.get_compute(id)
         elif depl_type in [DeploymentType.CSERVE_V2, DeploymentType.CSERVE_V3]:
@@ -169,21 +170,18 @@ def get(type, id):
         )
 
         click.echo("Additional deployment configurations:")
-        if depl_type == DeploymentType.INFERENCE_V2:
-            click.echo(
-                tabulate(
-                    [
-                        ("Image", deployment.image_url),
-                        ("Container port", deployment.container_port),
-                        ("Healthcheck", deployment.healthcheck or "/"),
-                        ("Replicas", _get_replica_info(deployment)),
-                        ("Environment variables", deployment.env_vars or "None"),
-                        ("Max concurrency", deployment.concurrency or "None"),
-                    ],
-                    tablefmt="rounded_outline",
-                    disable_numparse=True,
-                )
-            )
+        if depl_type in [DeploymentType.INFERENCE_V2, DeploymentType.INFERENCE_V3]:
+            replica_info = _get_replica_info(deployment)
+            display_rows = [
+                ("Image", deployment.image_url),
+                ("Container port", deployment.container_port),
+                ("Healthcheck", deployment.healthcheck or "/"),
+                ("Replicas", replica_info),
+                ("Environment variables", deployment.env_vars or "None"),
+                ("Max concurrency", deployment.concurrency or "None"),
+            ]
+
+            click.echo(tabulate(display_rows, tablefmt="rounded_outline", disable_numparse=True))
         elif depl_type == DeploymentType.COMPUTE_V2:
             click.echo(
                 tabulate(
diff --git a/centml/sdk/api.py b/centml/sdk/api.py
@@ -6,6 +6,7 @@
     DeploymentType,
     DeploymentStatus,
     CreateInferenceDeploymentRequest,
+    CreateInferenceV3DeploymentRequest,
     CreateComputeDeploymentRequest,
     CreateCServeV2DeploymentRequest,
     CreateCServeV3DeploymentRequest,
@@ -30,7 +31,21 @@ def get_status(self, id):
         return self._api.get_deployment_status_deployments_status_deployment_id_get(id)
 
     def get_inference(self, id):
-        return self._api.get_inference_deployment_deployments_inference_deployment_id_get(id)
+        """Get Inference deployment details - automatically handles both V2 and V3 deployments"""
+        # Try V3 first (recommended), fallback to V2 if deployment is V2
+        try:
+            return self._api.get_inference_v3_deployment_deployments_inference_v3_deployment_id_get(id)
+        except ApiException as e:
+            # If V3 fails with 404 or similar, try V2
+            if e.status in [404, 400]:  # Deployment might be V2 or endpoint not found
+                try:
+                    return self._api.get_inference_deployment_deployments_inference_deployment_id_get(id)
+                except ApiException as v2_error:
+                    # If both fail, raise the original V3 error as it's more likely to be the real issue
+                    raise e
+            else:
+                # For other errors (auth, network, etc.), raise immediately
+                raise
 
     def get_compute(self, id):
         return self._api.get_compute_deployment_deployments_compute_deployment_id_get(id)
@@ -52,9 +67,15 @@ def get_cserve(self, id):
                 # For other errors (auth, network, etc.), raise immediately
                 raise
 
-    def create_inference(self, request: CreateInferenceDeploymentRequest):
+    def create_inference(self, request: CreateInferenceV3DeploymentRequest):
+        return self._api.create_inference_v3_deployment_deployments_inference_v3_post(request)
+
+    def create_inference_v2(self, request: CreateInferenceDeploymentRequest):
         return self._api.create_inference_deployment_deployments_inference_post(request)
 
+    def create_inference_v3(self, request: CreateInferenceV3DeploymentRequest):
+        return self._api.create_inference_v3_deployment_deployments_inference_v3_post(request)
+
     def create_compute(self, request: CreateComputeDeploymentRequest):
         return self._api.create_compute_deployment_deployments_compute_post(request)
 
@@ -67,8 +88,51 @@ def create_cserve_v2(self, request: CreateCServeV2DeploymentRequest):
     def create_cserve_v3(self, request: CreateCServeV3DeploymentRequest):
         return self._api.create_cserve_v3_deployment_deployments_cserve_v3_post(request)
 
-    def update_inference(self, deployment_id: int, request: CreateInferenceDeploymentRequest):
-        return self._api.update_inference_deployment_deployments_inference_put(deployment_id, request)
+    def detect_inference_deployment_version(self, deployment_id: int) -> str:
+        """Detect if an inference deployment is V2 or V3 by testing the specific API endpoints"""
+        try:
+            # Try V3 endpoint first
+            self._api.get_inference_v3_deployment_deployments_inference_v3_deployment_id_get(deployment_id)
+            return 'v3'
+        except ApiException as e:
+            if e.status in [404, 400]:  # V3 endpoint doesn't exist for this deployment
+                try:
+                    # Try V2 endpoint
+                    self._api.get_inference_deployment_deployments_inference_deployment_id_get(deployment_id)
+                    return 'v2'
+                except ApiException:
+                    # If both fail, it might not be an inference deployment or doesn't exist
+                    raise ValueError(
+                        f"Deployment {deployment_id} is not a valid inference deployment or does not exist"
+                    )
+            else:
+                # Other error (auth, network, etc.)
+                raise
+
+    def update_inference(
+        self, deployment_id: int, request: Union[CreateInferenceDeploymentRequest, CreateInferenceV3DeploymentRequest]
+    ):
+        """Update Inference deployment - validates request type matches deployment version"""
+        # Detect the deployment version
+        deployment_version = self.detect_inference_deployment_version(deployment_id)
+
+        # Validate request type matches deployment version
+        if isinstance(request, CreateInferenceV3DeploymentRequest):
+            if deployment_version != 'v3':
+                raise ValueError(
+                    f"Deployment {deployment_id} is Inference {deployment_version.upper()}, but you provided a V3 request. Please use CreateInferenceDeploymentRequest instead."
+                )
+            return self._api.update_inference_v3_deployment_deployments_inference_v3_put(deployment_id, request)
+        elif isinstance(request, CreateInferenceDeploymentRequest):
+            if deployment_version != 'v2':
+                raise ValueError(
+                    f"Deployment {deployment_id} is Inference {deployment_version.upper()}, but you provided a V2 request. Please use CreateInferenceV3DeploymentRequest instead."
+                )
+            return self._api.update_inference_deployment_deployments_inference_put(deployment_id, request)
+        else:
+            raise ValueError(
+                f"Unsupported request type: {type(request)}. Expected CreateInferenceDeploymentRequest or CreateInferenceV3DeploymentRequest."
+            )
 
     def update_compute(self, deployment_id: int, request: CreateComputeDeploymentRequest):
         return self._api.update_compute_deployment_deployments_compute_put(deployment_id, request)
diff --git a/examples/sdk/create_inference.py b/examples/sdk/create_inference.py
@@ -1,26 +1,32 @@
 import centml
 from centml.sdk.api import get_centml_client
-from centml.sdk import DeploymentType, CreateInferenceDeploymentRequest, UserVaultType
+from centml.sdk import DeploymentType, CreateInferenceV3DeploymentRequest, UserVaultType
 
 
 def main():
     with get_centml_client() as cclient:
         certs = cclient.get_user_vault(UserVaultType.CERTIFICATES)
 
-        request = CreateInferenceDeploymentRequest(
+        request = CreateInferenceV3DeploymentRequest(
             name="nginx",
             cluster_id=1000,
             hardware_instance_id=1000,
             image_url="nginxinc/nginx-unprivileged",
             port=8080,
-            min_scale=1,
-            max_scale=1,
+            min_replicas=1,  # V3 uses min_replicas instead of min_scale
+            max_replicas=3,  # V3 uses max_replicas instead of max_scale
+            initial_replicas=1,  # Optional in V3 - initial number of replicas
             endpoint_certificate_authority=certs["my_cert"],
+            # V3 rollout strategy parameters
+            max_surge=1,  # Allow 1 extra pod during updates
+            max_unavailable=0,  # Keep all pods available during updates
+            healthcheck="/",
+            concurrency=10,
         )
         response = cclient.create_inference(request)
         print("Create deployment response: ", response)
 
-        ### Get deployment details
+        ### Get deployment details (automatically detects V2 or V3)
         deployment = cclient.get_inference(response.id)
         print("Deployment details: ", deployment)
 
