[Model] Fix Skywork R1V mlp (#26673)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Author: jeejeelee

vllm/model_executor/models/skyworkr1v.py

@@ -691,7 +691,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
             prefix=maybe_prefix(prefix, "language_model"),
         )
 
-        self.mlp1 = self._init_mlp1(config)
+        self.mlp1 = self._init_mlp1(
+            config, quant_config, prefix=maybe_prefix(prefix, "mlp1")
+        )
 
         self.img_context_token_id = None
         self.visual_token_mask = None
@@ -738,7 +740,12 @@ def _init_vision_model(
         else:
             return InternVisionPatchModel(config.vision_config)
 
-    def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
+    def _init_mlp1(
+        self,
+        config: PretrainedConfig,
+        quant_config: QuantizationConfig,
+        prefix: str = "",
+    ) -> nn.Module:
         vit_hidden_size = config.vision_config.hidden_size
         llm_hidden_size = config.text_config.hidden_size
 
@@ -748,9 +755,17 @@ def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
                 vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
                 llm_hidden_size,
                 return_bias=False,
+                quant_config=quant_config,
+                prefix=f"{prefix}.1",
             ),
             nn.GELU(),
-            ReplicatedLinear(llm_hidden_size, llm_hidden_size, return_bias=False),
+            ReplicatedLinear(
+                llm_hidden_size,
+                llm_hidden_size,
+                return_bias=False,
+                quant_config=quant_config,
+                prefix=f"{prefix}.3",
+            ),
         )
 
     def pixel_shuffle(self, x, scale_factor=0.5):