From 9f037e70753435478f73053d6442e5087d028678 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 11 Aug 2025 19:41:32 +0800 Subject: [PATCH 1/6] add auto config kwargs overrides Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 6b70164c8caf..7583191d3510 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -97,6 +97,12 @@ def _get_hf_token() -> Optional[str]: "llm_config": "text_config", } +_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = { + "NVLM_D": { + "has_no_defaults_at_init": True + }, +} + class ConfigFormat(str, enum.Enum): AUTO = "auto" @@ -291,6 +297,16 @@ def is_interleaved(config: PretrainedConfig) -> bool: return False +def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], *, + model_type: str): + """ + Override default has_no_defaults_at_init for AutoConfig based on model_type + """ + if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES: + kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type]) + return kwargs + + def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig: """Remap config attributes to match the expected names.""" for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items(): @@ -408,15 +424,14 @@ def get_config( ) else: try: + kwargs = _maybe_update_auto_config_kwargs( + kwargs, model_type=model_type) config = AutoConfig.from_pretrained( model, trust_remote_code=trust_remote_code, revision=revision, code_revision=code_revision, token=_get_hf_token(), - # some old custom model's config needs - # `has_no_defaults_at_init=True` to work. - has_no_defaults_at_init=trust_remote_code, **kwargs, ) except ValueError as e: From d87852b18d13b442c2ace7ed1bc45f82e0b25c72 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 11 Aug 2025 20:38:27 +0800 Subject: [PATCH 2/6] remove nvlm_d config Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 7 ++--- vllm/transformers_utils/configs/__init__.py | 2 -- vllm/transformers_utils/configs/nvlm_d.py | 31 --------------------- 3 files changed, 2 insertions(+), 38 deletions(-) delete mode 100644 vllm/transformers_utils/configs/nvlm_d.py diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 7583191d3510..9d619758fb5d 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -34,9 +34,8 @@ KimiVLConfig, MedusaConfig, MllamaConfig, MLPSpeculatorConfig, Nemotron_Nano_VL_Config, - NemotronConfig, NVLM_D_Config, - OvisConfig, RWConfig, - SpeculatorsConfig, + NemotronConfig, OvisConfig, + RWConfig, SpeculatorsConfig, Step3TextConfig, Step3VLConfig, UltravoxConfig) # yapf: enable @@ -85,7 +84,6 @@ def _get_hf_token() -> Optional[str]: "eagle": EAGLEConfig, "speculators": SpeculatorsConfig, "nemotron": NemotronConfig, - "NVLM_D": NVLM_D_Config, "ovis": OvisConfig, "ultravox": UltravoxConfig, "step3_vl": Step3VLConfig, @@ -313,7 +311,6 @@ def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig: if hasattr(config, old_attr): if not hasattr(config, new_attr): config.update({new_attr: getattr(config, old_attr)}) - delattr(config, old_attr) logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr) return config diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 82d24bb16ba5..dfc75295131e 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -23,7 +23,6 @@ from vllm.transformers_utils.configs.nemotron import NemotronConfig from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig from vllm.transformers_utils.configs.nemotron_vl import Nemotron_Nano_VL_Config -from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config from vllm.transformers_utils.configs.ovis import OvisConfig from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig from vllm.transformers_utils.configs.step3_vl import (Step3TextConfig, @@ -45,7 +44,6 @@ "NemotronConfig", "NemotronHConfig", "Nemotron_Nano_VL_Config", - "NVLM_D_Config", "OvisConfig", "SpeculatorsConfig", "UltravoxConfig", diff --git a/vllm/transformers_utils/configs/nvlm_d.py b/vllm/transformers_utils/configs/nvlm_d.py deleted file mode 100644 index edfc506882ff..000000000000 --- a/vllm/transformers_utils/configs/nvlm_d.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://huggingface.co/nvidia/NVLM-D-72B/blob/main/configuration_nvlm_d.py -# -------------------------------------------------------- -# NVLM-D -# Copyright (c) 2024 NVIDIA -# Licensed under Apache 2.0 License [see LICENSE for details] -# -------------------------------------------------------- -from transformers import Qwen2Config -from transformers.configuration_utils import PretrainedConfig - - -class NVLM_D_Config(PretrainedConfig): - model_type = 'NVLM_D' - is_composition = True - - def __init__(self, vision_config=None, llm_config=None, **kwargs): - super().__init__(**kwargs) - - # Handle vision_config initialization - if vision_config is None: - vision_config = {} - - # Handle llm_config initialization - if llm_config is None: - llm_config = {} - - self.vision_config = PretrainedConfig(**vision_config) - self.text_config = Qwen2Config(**llm_config) From 9fe73f2e64b371463a546f7285aa99cf86ecf0f5 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 11 Aug 2025 21:01:08 +0800 Subject: [PATCH 3/6] clean mllama config Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 22 ++++++++------- vllm/transformers_utils/configs/__init__.py | 4 +-- vllm/transformers_utils/configs/mllama.py | 31 --------------------- 3 files changed, 13 insertions(+), 44 deletions(-) delete mode 100644 vllm/transformers_utils/configs/mllama.py diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 9d619758fb5d..efd0c273462d 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -32,7 +32,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DeepseekVLV2Config, EAGLEConfig, JAISConfig, KimiVLConfig, MedusaConfig, - MllamaConfig, MLPSpeculatorConfig, + MLPSpeculatorConfig, Nemotron_Nano_VL_Config, NemotronConfig, OvisConfig, RWConfig, SpeculatorsConfig, @@ -67,10 +67,6 @@ def _get_hf_token() -> Optional[str]: return None -_CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = { - "mllama": MllamaConfig -} - _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = { "chatglm": ChatGLMConfig, "deepseek_vl_v2": DeepseekVLV2Config, @@ -88,7 +84,6 @@ def _get_hf_token() -> Optional[str]: "ultravox": UltravoxConfig, "step3_vl": Step3VLConfig, "step3_text": Step3TextConfig, - **_CONFIG_REGISTRY_OVERRIDE_HF } _CONFIG_ATTRS_MAPPING: dict[str, str] = { @@ -96,6 +91,12 @@ def _get_hf_token() -> Optional[str]: } _AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = { + "internvl_chat": { + "has_no_defaults_at_init": True + }, + "mllama": { + "is_encoder_decoder": True + }, "NVLM_D": { "has_no_defaults_at_init": True }, @@ -277,11 +278,12 @@ def thinker_uses_mrope(config: PretrainedConfig) -> bool: def is_encoder_decoder(config: PretrainedConfig) -> bool: """Detect if the model with this config is used as an encoder/decoder.""" - text_config = getattr(config, "text_config", None) - if text_config is not None: - return is_encoder_decoder(text_config) - return getattr(config, "is_encoder_decoder", False) + def _is_encoder_decoder(config: PretrainedConfig) -> bool: + return getattr(config, "is_encoder_decoder", False) + + return (_is_encoder_decoder(config) + or _is_encoder_decoder(config.get_text_config())) def is_interleaved(config: PretrainedConfig) -> bool: diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index dfc75295131e..8339c55bcf80 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -17,7 +17,6 @@ from vllm.transformers_utils.configs.jais import JAISConfig from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig from vllm.transformers_utils.configs.medusa import MedusaConfig -from vllm.transformers_utils.configs.mllama import MllamaConfig from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig from vllm.transformers_utils.configs.moonvit import MoonViTConfig from vllm.transformers_utils.configs.nemotron import NemotronConfig @@ -33,11 +32,10 @@ __all__ = [ "ChatGLMConfig", "DeepseekVLV2Config", + "EAGLEConfig", "RWConfig", "JAISConfig", "MedusaConfig", - "EAGLEConfig", - "MllamaConfig", "MLPSpeculatorConfig", "MoonViTConfig", "KimiVLConfig", diff --git a/vllm/transformers_utils/configs/mllama.py b/vllm/transformers_utils/configs/mllama.py deleted file mode 100644 index f0cd2d52a529..000000000000 --- a/vllm/transformers_utils/configs/mllama.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers.models.mllama import configuration_mllama as mllama_hf_config - - -class MllamaTextConfig(mllama_hf_config.MllamaTextConfig): - ''' - Use this class to override is_encoder_decoder: - - transformers regards mllama as is_encoder_decoder=False - - vllm needs is_encoder_decoder=True to enable cross-attention - ''' - - def __init__( - self, - **kwargs, - ): - super().__init__(**kwargs) - self.is_encoder_decoder = True - - -class MllamaConfig(mllama_hf_config.MllamaConfig): - - def __init__( - self, - text_config=None, - **kwargs, - ): - if isinstance(text_config, dict): - text_config = MllamaTextConfig(**text_config) - super().__init__(text_config=text_config, **kwargs) From 122338baec036c71fbdf0753f0f32ee2bd660058 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 11 Aug 2025 21:28:32 +0800 Subject: [PATCH 4/6] clean Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index efd0c273462d..e38440288eb5 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -297,8 +297,7 @@ def is_interleaved(config: PretrainedConfig) -> bool: return False -def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], *, - model_type: str): +def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str): """ Override default has_no_defaults_at_init for AutoConfig based on model_type """ From cc288c2d3282ec64e65badba40a00313cf33af07 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 11 Aug 2025 21:33:27 +0800 Subject: [PATCH 5/6] add comment Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index e38440288eb5..f51ebfbb994e 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -94,6 +94,8 @@ def _get_hf_token() -> Optional[str]: "internvl_chat": { "has_no_defaults_at_init": True }, + # transformers regards mllama as is_encoder_decoder=False + # vllm needs is_encoder_decoder=True to enable cross-attention "mllama": { "is_encoder_decoder": True }, From 55ab0458ae06f791b1edcbb42325edad35b1c991 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 11 Aug 2025 21:35:12 +0800 Subject: [PATCH 6/6] fix docstring Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index f51ebfbb994e..02ea0814ddef 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -301,7 +301,7 @@ def is_interleaved(config: PretrainedConfig) -> bool: def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str): """ - Override default has_no_defaults_at_init for AutoConfig based on model_type + Update kwargs for AutoConfig initialization based on model_type """ if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES: kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])