Remove hardcode flash-attn disable setting (#2342)

lm-sys · Sep 1, 2023 · 2264580 · 2264580
1 parent 2fbfcbc
commit 2264580
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/fastchat/model/model_adapter.py b/fastchat/model/model_adapter.py
@@ -1339,7 +1339,8 @@ def load_model(self, model_path: str, from_pretrained_kwargs: dict):
             model_path,
             trust_remote_code=True,
         )
-        config.use_flash_attn = False
+        # NOTE: if you use the old version of model file, please remove the comments below
+        # config.use_flash_attn = False
         config.fp16 = True
         generation_config = GenerationConfig.from_pretrained(
             model_path, trust_remote_code=True