File tree Expand file tree Collapse file tree 1 file changed +4
-2
lines changed Expand file tree Collapse file tree 1 file changed +4
-2
lines changed Original file line number Diff line number Diff line change @@ -53,10 +53,12 @@ def test_defaults_with_usage_context():
5353 vllm_config : VllmConfig = engine_args .create_engine_config (UsageContext .LLM_CLASS )
5454
5555 from vllm .platforms import current_platform
56+ from vllm .utils .mem_constants import GiB_bytes
5657
58+ device_memory = current_platform .get_device_total_memory ()
5759 device_name = current_platform .get_device_name ().lower ()
58- if "h100" in device_name or "h200" in device_name :
59- # For H100 and H200, we use larger default values.
60+ if device_memory >= 70 * GiB_bytes and "a100" not in device_name :
61+ # For GPUs like H100, H200, and MI300x with >= 70GB memory
6062 default_llm_tokens = 16384
6163 default_server_tokens = 8192
6264 default_max_num_seqs = 1024
You can’t perform that action at this time.
0 commit comments