[0.9.1][Fixbug] Fix num_hidden_layers when Qwen2-Audio and Qwen2.5-Omni

Signed-off-by: hfadzxy <starmoon_zhang@163.com>

Author: zhangxinyuehfad

docs/source/user_guide/support_matrix/supported_models.md

@@ -46,6 +46,7 @@ Get the newest info here: https://github.com/vllm-project/vllm-ascend/issues/160
 | InternVL2 | ✅ | |
 | InternVL2.5 | ✅ | |
 | Qwen2-Audio | ✅ |  |
+| Qwen2.5-Omni | ✅ | |
 | LLaVA-Next |  | Need test |
 | LLaVA-Next-Video | | Need test |
 | Phi-3-Vison/Phi-3.5-Vison | | Need test |

requirements-dev.txt

@@ -13,3 +13,5 @@ types-psutil
 networkx
 ray>=2.47.1
 protobuf>3.20.0
+librosa
+soundfile

tests/singlecard/test_offline_inference.py

@@ -27,6 +27,7 @@
 import vllm  # noqa: F401
 from modelscope import snapshot_download  # type: ignore[import-untyped]
 from vllm import SamplingParams
+from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
 
 import vllm_ascend  # noqa: F401
@@ -36,7 +37,8 @@
     "Qwen/Qwen2.5-0.5B-Instruct",
     "Qwen/Qwen3-0.6B-Base",
 ]
-MULTIMODALITY_MODELS = ["Qwen/Qwen2.5-VL-3B-Instruct"]
+MULTIMODALITY_VL_MODELS = ["Qwen/Qwen2.5-VL-3B-Instruct"]
+MULTIMODALITY_AUDIO_MODELS = ["Qwen/Qwen2-Audio-7B-Instruct"]
 
 QUANTIZATION_MODELS = [
     "vllm-ascend/Qwen2.5-0.5B-Instruct-W8A8",
@@ -47,6 +49,11 @@
     "vllm-ascend/Qwen2.5-0.5B-Instruct-W8A8",
 ]
 
+AUDIO_ASSETS = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
+AUDIO_PROMPT_TEMPLATES = {
+    1: "What is recited in the audio?",
+    2: "What sport and what nursery rhyme are referenced?"
+}
 
 @pytest.mark.parametrize("model", MODELS)
 @pytest.mark.parametrize("dtype", ["half", "float16"])
@@ -88,8 +95,8 @@ def test_quantization_models(model: str, max_tokens: int) -> None:
         vllm_model.generate_greedy(example_prompts, max_tokens)
 
 
-@pytest.mark.parametrize("model", MULTIMODALITY_MODELS)
-def test_multimodal(model, prompt_template, vllm_runner):
+@pytest.mark.parametrize("model", MULTIMODALITY_VL_MODELS)
+def test_multimodal_vl(model, prompt_template, vllm_runner):
     image = ImageAsset("cherry_blossom") \
         .pil_image.convert("RGB")
     img_questions = [
@@ -131,3 +138,41 @@ def test_models_topk() -> None:
                     enforce_eager=True,
                     gpu_memory_utilization=0.7) as vllm_model:
         vllm_model.generate(example_prompts, sampling_params)
+
+def prepare_audio_inputs(audio_count: int):
+    audio_prompt = "".join([
+        f"Audio {idx+1}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+        for idx in range(audio_count)
+    ])
+    question = AUDIO_PROMPT_TEMPLATES[audio_count]
+    prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+              "<|im_start|>user\n"
+              f"{audio_prompt}{question}<|im_end|>\n"
+              "<|im_start|>assistant\n")
+    mm_data = {
+        "audio":
+        [asset.audio_and_sample_rate for asset in AUDIO_ASSETS[:audio_count]]
+    }
+    inputs = {"prompt": prompt, "multi_modal_data": mm_data}
+    return inputs
+
+
+@pytest.mark.parametrize("model", MULTIMODALITY_AUDIO_MODELS)
+@pytest.mark.parametrize("audio_count", [2])
+@pytest.mark.parametrize("max_tokens", [10])
+def test_multimodal_audio(model: str, audio_count: int,
+                          max_tokens: int) -> None:
+    inputs = prepare_audio_inputs(audio_count)
+
+    sampling_params = SamplingParams(temperature=0.2,
+                                     max_tokens=max_tokens,
+                                     stop_token_ids=None)
+
+    with VllmRunner(model,
+                    max_model_len=4096,
+                    max_num_seqs=5,
+                    enforce_eager=False,
+                    dtype="bfloat16",
+                    limit_mm_per_prompt={"audio": audio_count},
+                                        gpu_memory_utilization=0.9) as vllm_model:
+        vllm_model.generate(inputs, sampling_params=sampling_params)

vllm_ascend/utils.py

@@ -175,6 +175,27 @@ def vllm_version_is(target_vllm_version: str):
             "format of x.y.z.")
 
 
+def get_max_hidden_layers(hf_config) -> int:
+    cfg_dict = hf_config.to_dict()
+    layer_counts = []
+
+    def _rec_find(d):
+        if isinstance(d, dict):
+            for k, v in d.items():
+                if k == "num_hidden_layers" and isinstance(v, int):
+                    layer_counts.append(v)
+                else:
+                    _rec_find(v)
+        elif isinstance(d, list):
+            for item in d:
+                _rec_find(item)
+
+    _rec_find(cfg_dict)
+    if not layer_counts:
+        raise ValueError("Not found num_hidden_layers in model config.")
+    return max(layer_counts)
+
+
 def update_aclgraph_sizes(vllm_config: VllmConfig) -> None:
     """Update ACL graph capture sizes based on hardware limitations"""
     # Store original configuration and temporarily clear it
@@ -204,7 +225,11 @@ def update_aclgraph_sizes(vllm_config: VllmConfig) -> None:
         return
 
     # Calculate parallel configuration factor
-    num_hidden_layers = vllm_config.model_config.hf_config.num_hidden_layers
+    hf_config = vllm_config.model_config.hf_config
+    if hasattr(hf_config, 'num_hidden_layers'):
+        num_hidden_layers = hf_config.num_hidden_layers
+    else:
+        num_hidden_layers = get_max_hidden_layers(hf_config)
     parallel_config = vllm_config.parallel_config
 
     # TODO: Find out whether we need to take into account the pp_size

vllm_ascend/worker/model_runner.py

@@ -866,6 +866,8 @@ def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
                 "mrope embedding type requires multi-modal input mapper "
                 "returns 'image_grid_thw' or 'video_grid_thw'.")
             second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
+            audio_feature_lengths= mm_kwargs.get("audio_feature_lengths", None)
+            use_audio_in_video=mm_kwargs.get("use_audio_in_video", False)
 
             hf_config = self.runner.model_config.hf_config
 
@@ -884,6 +886,8 @@ def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
                         second_per_grid_ts=second_per_grid_ts,
                         context_len=inter_data.context_lens[seq_idx],
                         seq_len=inter_data.seq_lens[seq_idx],
+                        audio_feature_lengths=audio_feature_lengths,
+                        use_audio_in_video=use_audio_in_video,
                     )
 
                 seq_data.mrope_position_delta = mrope_position_delta