fix model overrides dict

theroyallab · Sep 26, 2024 · 7de70ec · 7de70ec
1 parent 55cf8b6
commit 7de70ec
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 5 deletions.
diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py
@@ -339,8 +339,6 @@ async def create(
 
         # Set user-configured draft model values
         if draft.draft_model_name:
-            # Fetch from the updated kwargs
-            draft_args = unwrap(draft, {})
 
             self.draft_config.max_seq_len = self.config.max_seq_len
 

diff --git a/backends/exllamav2/types.py b/backends/exllamav2/types.py
@@ -38,6 +38,8 @@ class DraftModelInstanceConfig(BaseModel):
         ),
     )
 
+    model_config = ConfigDict(revalidate_instances="always")
+
 
 class ModelInstanceConfig(BaseModel):
     """
@@ -182,4 +184,4 @@ class ModelInstanceConfig(BaseModel):
         ge=1,
     )
 
-    model_config = ConfigDict(protected_namespaces=())
+    model_config = ConfigDict(protected_namespaces=(), revalidate_instances="always")
diff --git a/common/model.py b/common/model.py
@@ -14,6 +14,7 @@
 from common.logger import get_loading_progress_bar
 from common.networking import handle_request_error
 from common.optional_dependencies import dependencies
+from common.tabby_config import config
 
 if dependencies.exllamav2:
     from backends.exllamav2.model import ExllamaV2Container
@@ -69,8 +70,8 @@ async def load_model_gen(
         await unload_model()
 
     # Merge with config defaults
-    # FIXME: KWARGS DO NOT EXIST NOW
-    # kwargs = {**config.model_defaults, **kwargs}
+    model = model.model_copy(update=config.model_defaults)
+    model.model_validate(model, strict=True)
 
     # Create a new container
     draft = draft or DraftModelInstanceConfig()