diff --git a/examples/sentiment/scripts/gpt-neox-20b_peft/clm_finetune_peft_imdb.py b/examples/sentiment/scripts/gpt-neox-20b_peft/clm_finetune_peft_imdb.py
index b78111605c..7ce3cc400f 100644
--- a/examples/sentiment/scripts/gpt-neox-20b_peft/clm_finetune_peft_imdb.py
+++ b/examples/sentiment/scripts/gpt-neox-20b_peft/clm_finetune_peft_imdb.py
@@ -59,9 +59,7 @@ class DataTrainingArguments:
 
 
 if "gpt-neox" in model_args.model_name_or_path:
-    model = prepare_model_for_int8_training(
-        model, output_embedding_layer_name="embed_out", layer_norm_names=["layer_norm", "layernorm"]
-    )
+    model = prepare_model_for_int8_training(model, output_embedding_layer_name="embed_out")
 else:
     model = prepare_model_for_int8_training(model)
 
diff --git a/examples/sentiment/scripts/gpt2-sentiment_peft.py b/examples/sentiment/scripts/gpt2-sentiment_peft.py
index 98274864b0..fbb2b23947 100644
--- a/examples/sentiment/scripts/gpt2-sentiment_peft.py
+++ b/examples/sentiment/scripts/gpt2-sentiment_peft.py
@@ -152,7 +152,6 @@ def collator(data):
     config.model_name,
     load_in_8bit=True,
     peft_config=lora_config,
-    layer_norm_names=[],
 )
 
 tokenizer = AutoTokenizer.from_pretrained(config.model_name)
diff --git a/examples/stack_llama/scripts/rl_training.py b/examples/stack_llama/scripts/rl_training.py
index f38c654f73..0d8c9b1ce8 100644
--- a/examples/stack_llama/scripts/rl_training.py
+++ b/examples/stack_llama/scripts/rl_training.py
@@ -183,7 +183,6 @@ def collator(data):
     load_in_8bit=True,
     device_map={"": current_device},
     peft_config=lora_config,
-    layer_norm_names=[],
 )
 
 optimizer = None
diff --git a/requirements.txt b/requirements.txt
index 8c7d9c6c25..b4c9eed937 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ torch>=1.4.0
 tqdm
 transformers
 accelerate
+peft>=0.3.0