From 72bb488a24ec000486af362fff9cda62e0571269 Mon Sep 17 00:00:00 2001
From: Arnav Garg <arnav@predibase.com>
Date: Wed, 21 Feb 2024 13:37:25 -0800
Subject: [PATCH] Add default LoRA and IA3 target modules for Gemma

---
 src/peft/utils/constants.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/peft/utils/constants.py b/src/peft/utils/constants.py
index daef39cff5..41047431ce 100644
--- a/src/peft/utils/constants.py
+++ b/src/peft/utils/constants.py
@@ -76,6 +76,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "mixtral": ["q_proj", "v_proj"],
     "stablelm": ["q_proj", "v_proj"],
     "phi": ["q_proj", "v_proj", "fc1", "fc2"],
+    "gemma": ["q_proj", "v_proj"],
 }
 
 TRANSFORMERS_MODELS_TO_IA3_TARGET_MODULES_MAPPING = {
@@ -100,6 +101,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "RefinedWeb": ["query_key_value", "dense_4h_to_h"],
     "falcon": ["query_key_value", "dense_4h_to_h"],
     "phi": ["q_proj", "v_proj", "fc2"],
+    "gemma": ["q_proj", "v_proj", "down_proj"],
 }
 
 TRANSFORMERS_MODELS_TO_IA3_FEEDFORWARD_MODULES_MAPPING = {
@@ -124,6 +126,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "RefinedWebModel": ["dense_4h_to_h"],
     "falcon": ["dense_4h_to_h"],
     "phi": ["fc2"],
+    "gemma": ["down_proj"],
 }
 
 TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING = {