[CI/Build] Fix LoRA OOM (vllm-project#16624)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Yang Wang <elainewy@meta.com>

Author: jeejeelee

tests/lora/test_minicpmv_tp.py

@@ -66,8 +66,12 @@ def test_minicpmv_lora(minicpmv_lora_files):
         max_loras=2,
         max_lora_rank=8,
         enforce_eager=True,
+        max_model_len=2048,
+        limit_mm_per_prompt={
+            "image": 2,
+            "video": 0
+        },
         trust_remote_code=True,
-        enable_chunked_prefill=True,
     )
     output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
     for i in range(len(EXPECTED_OUTPUT)):
@@ -91,9 +95,11 @@ def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):
         max_loras=4,
         max_lora_rank=64,
         tensor_parallel_size=4,
+        limit_mm_per_prompt={
+            "image": 2,
+            "video": 0
+        },
         trust_remote_code=True,
-        enforce_eager=True,
-        enable_chunked_prefill=True,
     )
     output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
     for i in range(len(EXPECTED_OUTPUT)):
@@ -115,8 +121,11 @@ def test_minicpmv_tp4_fully_sharded_loras(minicpmv_lora_files):
         max_lora_rank=8,
         tensor_parallel_size=4,
         trust_remote_code=True,
+        limit_mm_per_prompt={
+            "image": 1,
+            "video": 0
+        },
         fully_sharded_loras=True,
-        enable_chunked_prefill=True,
     )
     output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
     for i in range(len(EXPECTED_OUTPUT)):