Add default IA3 target modules for Mixtral (huggingface#1376)

* Add default LoRA target modules for Mixtral

* Add IA3 modules for Mixtral

* Address comments

Author: arnavgarg1

src/peft/utils/constants.py

@@ -93,6 +93,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "gpt_bigcode": ["c_attn", "mlp.c_proj"],
     "llama": ["k_proj", "v_proj", "down_proj"],
     "mistral": ["k_proj", "v_proj", "down_proj"],
+    "mixtral": ["k_proj", "v_proj", "w2"],
     "bert": ["key", "value", "output.dense"],
     "deberta-v2": ["key_proj", "value_proj", "output.dense"],
     "deberta": ["in_proj", "output.dense"],
@@ -116,6 +117,7 @@ def starcoder_model_postprocess_past_key_value(past_key_values):
     "gpt_bigcode": ["mlp.c_proj"],
     "llama": ["down_proj"],
     "mistral": ["down_proj"],
+    "mixtral": ["w2"],
     "bert": ["output.dense"],
     "deberta-v2": ["output.dense"],
     "deberta": ["output.dense"],