dont use get_peft_model if model is already peft (#857)

trl/trainer/reward_trainer.py

@@ -126,10 +126,13 @@ def __init__(
                 "PEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT models"
             )
         elif is_peft_available() and peft_config is not None:
-            if getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_quantized", False):
-                model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=args.gradient_checkpointing)
+            if not isinstance(model, PeftModel):
+                if getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_quantized", False):
+                    model = prepare_model_for_kbit_training(
+                        model, use_gradient_checkpointing=args.gradient_checkpointing
+                    )
 
-            model = get_peft_model(model, peft_config)
+                model = get_peft_model(model, peft_config)
 
         if is_peft_available() and callbacks is None and isinstance(model, PeftModel):
             callbacks = [PeftSavingCallback()]