Add IA3 Modules for Phi (#1407)

* Add IA3 Modules for Phi * Address comments
huggingface · Jan 30, 2024 · 68b90a1 · 68b90a1
1 parent 75e4ef3
commit 68b90a1
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/peft/utils/constants.py b/src/peft/utils/constants.py
@@ -99,6 +99,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "RefinedWebModel": ["query_key_value", "dense_4h_to_h"],
     "RefinedWeb": ["query_key_value", "dense_4h_to_h"],
     "falcon": ["query_key_value", "dense_4h_to_h"],
+    "phi": ["q_proj", "v_proj", "fc2"],
 }
 
 TRANSFORMERS_MODELS_TO_IA3_FEEDFORWARD_MODULES_MAPPING = {
@@ -121,6 +122,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "RefinedWeb": ["dense_4h_to_h"],
     "RefinedWebModel": ["dense_4h_to_h"],
     "falcon": ["dense_4h_to_h"],
+    "phi": ["fc2"],
 }
 
 TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING = {