[TRTLLM-6577][feat] Support nano_v2_vlm in pytorch backend

* support cache reuse.

Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com>

Author: Wanli-Jiang

docs/source/reference/multimodal-feature-support-matrix.md

@@ -8,6 +8,7 @@
 | LLaVA-NeXT         | Yes        | Yes                 | Yes            | No              |
 | Llama 4            | Yes        | Yes                 | No             | No              |
 | Mistral-Small-3.1  | Yes        | Yes                 | No             | No              |
+| Nano-v2-VLM        | Yes        | Yes                 | Yes            | No              |
 | Phi-4-multimodal   | Yes        | Yes                 | No             | No              |
 | Qwen2-VL           | Yes        | Yes                 | Yes            | No              |
 | Qwen2.5-VL         | Yes        | Yes                 | Yes            | No              |

tensorrt_llm/_torch/models/modeling_nanov2vlm.py

@@ -20,7 +20,8 @@
 from ..attention_backend import AttentionMetadata
 from ..model_config import ModelConfig
 from .modeling_auto import AutoModelForCausalLM
-from .modeling_multimodal_utils import fuse_input_embeds
+from .modeling_multimodal_utils import (find_uncached_mm_embeds,
+                                        fuse_input_embeds)
 from .modeling_radio import RADIOVisionModel
 from .modeling_utils import register_auto_model
 
@@ -394,6 +395,8 @@ def forward(
                     multimodal_param.multimodal_data["multimodal_embedding"]
                     for multimodal_param in multimodal_params
                 ]
+            mm_embedding = find_uncached_mm_embeds(
+                mm_embedding, multimodal_params[:num_context_requests])
         input_ids, input_embeds = fuse_input_embeds(
             self.llm.model.embed_tokens,
             input_ids,