[Bugfix] Prioritize dtype in root config before checking text config (vllm-project#17629)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Signed-off-by: Yuqi Zhang <yuqizhang@google.com>

Author: DarkLight1337

vllm/config.py

@@ -2954,10 +2954,12 @@ def _get_and_verify_dtype(
 ) -> torch.dtype:
     # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
     # because config.torch_dtype can be None.
-    config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
+    config_dtype = getattr(config, "torch_dtype", None)
 
-    # Fallback for multi-modal models if the root config
+    # Fallbacks for multi-modal models if the root config
     # does not define torch_dtype
+    if config_dtype is None:
+        config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
     if config_dtype is None and hasattr(config, "vision_config"):
         config_dtype = getattr(config.vision_config, "torch_dtype", None)