|
| 1 | +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. |
| 2 | +# Copyright 2023 The vLLM team. |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | +# This file is a part of the vllm-ascend project. |
| 16 | +# |
| 17 | +import json |
| 18 | +from typing import Any |
| 19 | + |
| 20 | +import openai |
| 21 | +import pytest |
| 22 | +from vllm.utils import get_open_port |
| 23 | + |
| 24 | +from tests.e2e.conftest import RemoteOpenAIServer |
| 25 | +from tools.aisbench import run_aisbench_cases |
| 26 | + |
| 27 | +MODELS = [ |
| 28 | + "vllm-ascend/DeepSeek-R1-0528-W8A8", |
| 29 | +] |
| 30 | + |
| 31 | +MODES = [ |
| 32 | + "torchair", |
| 33 | + "single", |
| 34 | + "aclgraph", |
| 35 | + "no_chunkprefill", |
| 36 | +] |
| 37 | + |
| 38 | +prompts = [ |
| 39 | + "San Francisco is a", |
| 40 | +] |
| 41 | + |
| 42 | +api_keyword_args = { |
| 43 | + "max_tokens": 10, |
| 44 | +} |
| 45 | + |
| 46 | +aisbench_cases = [{ |
| 47 | + "case_type": "accuracy", |
| 48 | + "dataset_path": "vllm-ascend/gsm8k-lite", |
| 49 | + "request_conf": "vllm_api_general_chat", |
| 50 | + "dataset_conf": "gsm8k/gsm8k_gen_0_shot_cot_chat_prompt", |
| 51 | + "max_out_len": 32768, |
| 52 | + "batch_size": 32, |
| 53 | + "baseline": 95, |
| 54 | + "threshold": 5 |
| 55 | +}, { |
| 56 | + "case_type": "performance", |
| 57 | + "dataset_path": "vllm-ascend/GSM8K-in3500-bs400", |
| 58 | + "request_conf": "vllm_api_stream_chat", |
| 59 | + "dataset_conf": "gsm8k/gsm8k_gen_0_shot_cot_str_perf", |
| 60 | + "num_prompts": 400, |
| 61 | + "max_out_len": 1500, |
| 62 | + "batch_size": 1000, |
| 63 | + "baseline": 1, |
| 64 | + "threshold": 0.97 |
| 65 | +}] |
| 66 | + |
| 67 | + |
| 68 | +@pytest.mark.asyncio |
| 69 | +@pytest.mark.parametrize("model", MODELS) |
| 70 | +@pytest.mark.parametrize("mode", MODES) |
| 71 | +async def test_models(model: str, mode: str) -> None: |
| 72 | + port = get_open_port() |
| 73 | + env_dict = { |
| 74 | + "OMP_NUM_THREADS": "10", |
| 75 | + "OMP_PROC_BIND": "false", |
| 76 | + "HCCL_BUFFSIZE": "1024", |
| 77 | + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True" |
| 78 | + } |
| 79 | + speculative_config = { |
| 80 | + "num_speculative_tokens": 1, |
| 81 | + "method": "deepseek_mtp" |
| 82 | + } |
| 83 | + additional_config = { |
| 84 | + "ascend_scheduler_config": { |
| 85 | + "enabled": False |
| 86 | + }, |
| 87 | + "torchair_graph_config": { |
| 88 | + "enabled": True, |
| 89 | + "enable_multistream_moe": False, |
| 90 | + "enable_multistream_mla": True, |
| 91 | + "graph_batch_sizes": [16], |
| 92 | + "use_cached_graph": True |
| 93 | + }, |
| 94 | + "chunked_prefill_for_mla": True, |
| 95 | + "enable_weight_nz_layout": True |
| 96 | + } |
| 97 | + server_args = [ |
| 98 | + "--quantization", "ascend", "--data-parallel-size", "2", |
| 99 | + "--tensor-parallel-size", "8", "--enable-expert-parallel", "--port", |
| 100 | + str(port), "--seed", "1024", "--max-model-len", "36864", |
| 101 | + "--max-num-batched-tokens", "4096", "--max-num-seqs", "16", |
| 102 | + "--trust-remote-code", "--gpu-memory-utilization", "0.9", |
| 103 | + "--speculative-config", |
| 104 | + json.dumps(speculative_config) |
| 105 | + ] |
| 106 | + if mode == "single": |
| 107 | + server_args.append("--enforce-eager") |
| 108 | + additional_config["torchair_graph_config"] = {"enabled": False} |
| 109 | + if mode == "aclgraph": |
| 110 | + additional_config["torchair_graph_config"] = {"enabled": False} |
| 111 | + if mode == "no_chunkprefill": |
| 112 | + additional_config["ascend_scheduler_config"] = {"enabled": True} |
| 113 | + i = server_args.index("--max-num-batched-tokens") + 1 |
| 114 | + server_args[i] = "36864" |
| 115 | + server_args.extend(["--additional-config", json.dumps(additional_config)]) |
| 116 | + request_keyword_args: dict[str, Any] = { |
| 117 | + **api_keyword_args, |
| 118 | + } |
| 119 | + with RemoteOpenAIServer(model, |
| 120 | + server_args, |
| 121 | + server_port=port, |
| 122 | + env_dict=env_dict, |
| 123 | + auto_port=False) as server: |
| 124 | + client = server.get_async_client() |
| 125 | + batch = await client.completions.create( |
| 126 | + model=model, |
| 127 | + prompt=prompts, |
| 128 | + **request_keyword_args, |
| 129 | + ) |
| 130 | + choices: list[openai.types.CompletionChoice] = batch.choices |
| 131 | + assert choices[0].text, "empty response" |
| 132 | + print(choices) |
| 133 | + if mode in ["single", "no_chunkprefill"]: |
| 134 | + return |
| 135 | + # aisbench test |
| 136 | + run_aisbench_cases(model, port, aisbench_cases) |
0 commit comments