Add falcon to officially supported LoRA & IA3 modules (#722)

* add falcon to officially supported modules * add lora * add also `RefinedWeb`
huggingface · Jul 19, 2023 · 3040782 · 3040782
1 parent 1b8b17d
commit 3040782
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/src/peft/utils/other.py b/src/peft/utils/other.py
@@ -281,6 +281,9 @@ def transpose(weight, fan_in_fan_out):
     "chatglm": ["query_key_value"],
     "gpt_bigcode": ["c_attn"],
     "mpt": ["Wqkv"],
+    "RefinedWebModel": ["query_key_value"],
+    "RefinedWeb": ["query_key_value"],
+    "falcon": ["query_key_value"],
 }
 
 TRANSFORMERS_MODELS_TO_IA3_TARGET_MODULES_MAPPING = {
@@ -299,6 +302,9 @@ def transpose(weight, fan_in_fan_out):
     "bert": ["key", "value", "output.dense"],
     "deberta-v2": ["key_proj", "value_proj", "output.dense"],
     "deberta": ["in_proj", "output.dense"],
+    "RefinedWebModel": ["query_key_value"],
+    "RefinedWeb": ["query_key_value"],
+    "falcon": ["query_key_value"],
 }
 
 TRANSFORMERS_MODELS_TO_IA3_FEEDFORWARD_MODULES_MAPPING = {
@@ -317,6 +323,9 @@ def transpose(weight, fan_in_fan_out):
     "bert": ["output.dense"],
     "deberta-v2": ["output.dense"],
     "deberta": ["output.dense"],
+    "RefinedWeb": ["query_key_value"],
+    "RefinedWebModel": ["query_key_value"],
+    "falcon": ["query_key_value"],
 }
 
 COMMON_LAYERS_PATTERN = ["layers", "h", "block", "blocks", "layer"]