Skip to content

Commit 842ae65

Browse files
feat: add multimodal deployment example for llava based on vllm v1 #2628 (#2672)
Co-authored-by: Dmitry Tokarev <dtokarev@nvidia.com>
1 parent 1c32e8f commit 842ae65

File tree

1 file changed

+68
-0
lines changed

1 file changed

+68
-0
lines changed
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: nvidia.com/v1alpha1
5+
kind: DynamoGraphDeployment
6+
metadata:
7+
name: agg-llava
8+
spec:
9+
backendFramework: vllm
10+
services:
11+
Frontend:
12+
dynamoNamespace: agg-llava
13+
componentType: frontend
14+
replicas: 1
15+
extraPodSpec:
16+
mainContainer:
17+
image: my-registry/vllm-runtime:my-tag
18+
EncodeWorker:
19+
envFromSecret: hf-token-secret
20+
dynamoNamespace: agg-llava
21+
componentType: worker
22+
replicas: 1
23+
resources:
24+
limits:
25+
gpu: "1"
26+
extraPodSpec:
27+
mainContainer:
28+
image: my-registry/vllm-runtime:my-tag
29+
workingDir: /workspace/examples/multimodal
30+
command:
31+
- /bin/sh
32+
- -c
33+
args:
34+
- python3 components/encode_worker.py --model llava-hf/llava-1.5-7b-hf
35+
VLMWorker:
36+
envFromSecret: hf-token-secret
37+
dynamoNamespace: agg-llava
38+
componentType: worker
39+
replicas: 1
40+
resources:
41+
limits:
42+
gpu: "1"
43+
extraPodSpec:
44+
mainContainer:
45+
image: my-registry/vllm-runtime:my-tag
46+
workingDir: /workspace/examples/multimodal
47+
command:
48+
- /bin/sh
49+
- -c
50+
args:
51+
- python3 components/worker.py --model llava-hf/llava-1.5-7b-hf --worker-type prefill
52+
Processor:
53+
envFromSecret: hf-token-secret
54+
dynamoNamespace: agg-llava
55+
componentType: worker
56+
replicas: 1
57+
resources:
58+
limits:
59+
gpu: "1"
60+
extraPodSpec:
61+
mainContainer:
62+
image: my-registry/vllm-runtime:my-tag
63+
workingDir: /workspace/examples/multimodal
64+
command:
65+
- /bin/sh
66+
- -c
67+
args:
68+
- 'python3 components/processor.py --model llava-hf/llava-1.5-7b-hf --prompt-template "USER: <image>\n<prompt> ASSISTANT:"'

0 commit comments

Comments
 (0)