[core] Fix peft multi-gpu issue

src/peft/tuners/lora.py

@@ -200,6 +200,11 @@ def _replace_module(self, parent_module, child_name, new_module, old_module):
             new_module.state = old_module.state
             new_module.to(old_module.weight.device)
 
+        # dispatch to correct device
+        for name, module in new_module.named_modules():
+            if "lora_" in name:
+                module.to(old_module.weight.device)
+
     def __getattr__(self, name: str):
         """Forward missing attributes to the wrapped module."""
         try:
@@ -339,17 +344,20 @@ def eval(self):
         self.lora_B.eval()
 
     def forward(self, x: torch.Tensor):
+
         if self.disable_adapters:
             if self.r > 0 and self.merged:
                 self.weight.data -= (
                     transpose(self.lora_B.weight @ self.lora_A.weight, self.fan_in_fan_out) * self.scaling
                 )
                 self.merged = False
+
             return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
         elif self.r > 0 and not self.merged:
             result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
             if self.r > 0:
                 result += self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling
+            result = result
             return result
         else:
             return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)