From 2831c6bfbb84e48f1fcff0b1fefdd49ee22a8147 Mon Sep 17 00:00:00 2001 From: Georg Slamanig Date: Thu, 27 Feb 2025 08:24:11 +0100 Subject: [PATCH 1/6] groups fix --- src/peft/tuners/lora/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 493b7b1852..7223c23132 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -1061,7 +1061,7 @@ def update_layer( conv_layer = type(base_layer) out_kernel = out_stride = (1,) * (self._kernel_dim - 2) self.lora_A[adapter_name] = conv_layer(self.in_features, r, kernel_size, stride, padding, bias=False) - self.lora_B[adapter_name] = conv_layer(r, self.out_features, out_kernel, out_stride, bias=lora_bias) + self.lora_B[adapter_name] = conv_layer(r, self.out_features // self.base_layer.groups, out_kernel, out_stride, bias=lora_bias) self.lora_bias[adapter_name] = lora_bias if use_rslora: From d960507c62b230ff0c32d8320c776d6eaf8dc9f4 Mon Sep 17 00:00:00 2001 From: Georg Slamanig Date: Thu, 27 Feb 2025 09:23:45 +0100 Subject: [PATCH 2/6] fixed merging error for groups layers --- src/peft/tuners/lora/layer.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 7223c23132..d3fbe2525e 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -1243,13 +1243,12 @@ def get_delta_weight(self, adapter) -> torch.Tensor: 3 ) * self.scaling[adapter] else: - output_tensor = ( - self.conv_fn( - weight_A.transpose(0, 1), - weight_B, - ).transpose(0, 1) - * self.scaling[adapter] - ) + output_tensor = self.conv_fn(weight_A.transpose(0, 1), weight_B) + + if self.base_layer.groups > 1: + output_tensor = output_tensor * self.scaling[adapter] + else: + output_tensor = output_tensor.transpose(0, 1) * self.scaling[adapter] if cast_to_fp32: output_tensor = output_tensor.to(dtype=dtype) From ea3c0f97fb4e3426e5b90739a242dbc2b26ee5d8 Mon Sep 17 00:00:00 2001 From: Georg Slamanig <148696483+gslama12@users.noreply.github.com> Date: Thu, 27 Feb 2025 12:02:56 +0100 Subject: [PATCH 3/6] Update src/peft/tuners/lora/layer.py Co-authored-by: Benjamin Bossan --- src/peft/tuners/lora/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index d3fbe2525e..5737eda4a3 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -1061,7 +1061,7 @@ def update_layer( conv_layer = type(base_layer) out_kernel = out_stride = (1,) * (self._kernel_dim - 2) self.lora_A[adapter_name] = conv_layer(self.in_features, r, kernel_size, stride, padding, bias=False) - self.lora_B[adapter_name] = conv_layer(r, self.out_features // self.base_layer.groups, out_kernel, out_stride, bias=lora_bias) + self.lora_B[adapter_name] = conv_layer(r, self.out_features // base_layer.groups, out_kernel, out_stride, bias=lora_bias) self.lora_bias[adapter_name] = lora_bias if use_rslora: From 675be66ffab4d2e3af2b8a9f2e7612687be7d7a9 Mon Sep 17 00:00:00 2001 From: Georg Slamanig <148696483+gslama12@users.noreply.github.com> Date: Thu, 27 Feb 2025 12:03:11 +0100 Subject: [PATCH 4/6] Update src/peft/tuners/lora/layer.py Co-authored-by: Benjamin Bossan --- src/peft/tuners/lora/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 5737eda4a3..5459a90bcf 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -1245,7 +1245,7 @@ def get_delta_weight(self, adapter) -> torch.Tensor: else: output_tensor = self.conv_fn(weight_A.transpose(0, 1), weight_B) - if self.base_layer.groups > 1: + if self.get_base_layer().groups > 1: output_tensor = output_tensor * self.scaling[adapter] else: output_tensor = output_tensor.transpose(0, 1) * self.scaling[adapter] From de4c2e437f3c025ab987402d8b7cdc8d222b7d04 Mon Sep 17 00:00:00 2001 From: Georg Slamanig Date: Thu, 27 Feb 2025 12:23:11 +0100 Subject: [PATCH 5/6] added test and model with groups --- src/peft/tuners/lora/layer.py | 4 +++- tests/test_custom_models.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 5459a90bcf..0460c8b3fb 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -1061,7 +1061,9 @@ def update_layer( conv_layer = type(base_layer) out_kernel = out_stride = (1,) * (self._kernel_dim - 2) self.lora_A[adapter_name] = conv_layer(self.in_features, r, kernel_size, stride, padding, bias=False) - self.lora_B[adapter_name] = conv_layer(r, self.out_features // base_layer.groups, out_kernel, out_stride, bias=lora_bias) + self.lora_B[adapter_name] = conv_layer( + r, self.out_features // base_layer.groups, out_kernel, out_stride, bias=lora_bias + ) self.lora_bias[adapter_name] = lora_bias if use_rslora: diff --git a/tests/test_custom_models.py b/tests/test_custom_models.py index 9813638e1a..ac0f2b34f4 100644 --- a/tests/test_custom_models.py +++ b/tests/test_custom_models.py @@ -115,6 +115,8 @@ ("Conv2d 2 LoRA", "Conv2d", LoraConfig, {"target_modules": ["conv2d", "lin0"]}), ("Conv2d 1 LoRA with DoRA", "Conv2d", LoraConfig, {"target_modules": ["conv2d"], "use_dora": True}), ("Conv2d 2 LoRA with DoRA", "Conv2d", LoraConfig, {"target_modules": ["conv2d", "lin0"], "use_dora": True}), + ("Conv2d Groups LoRA", "Conv2dGroups", LoraConfig, {"target_modules": ["conv2d"]}), + ("Conv2d Groups LoRA with DoRA", "Conv2dGroups", LoraConfig, {"target_modules": ["conv2d"], "use_dora": True}), ("Conv3d 1 LoRA", "Conv3d", LoraConfig, {"target_modules": ["conv3d"]}), ("Conv3d 2 LoRA", "Conv3d", LoraConfig, {"target_modules": ["conv3d", "lin0"]}), ("Conv3d 1 LoRA with DoRA", "Conv3d", LoraConfig, {"target_modules": ["conv3d"], "use_dora": True}), @@ -903,6 +905,25 @@ def forward(self, X): return X +class ModelConv2DGroups(nn.Module): + def __init__(self): + super().__init__() + self.conv2d = nn.Conv2d(5, 10, 3, groups=5) + self.relu = nn.ReLU() + self.flat = nn.Flatten() + self.lin0 = nn.Linear(10, 2) + self.sm = nn.LogSoftmax(dim=-1) + + def forward(self, X): + X = X.float().reshape(-1, 5, 3, 3) + X = self.conv2d(X) + X = self.relu(X) + X = self.flat(X) + X = self.lin0(X) + X = self.sm(X) + return X + + class ModelConv3D(nn.Module): def __init__(self): super().__init__() @@ -967,6 +988,9 @@ def from_pretrained(cls, model_id, torch_dtype=None): if model_id == "Conv2d": return ModelConv2D().to(torch_dtype) + if model_id == "Conv2dGroups": + return ModelConv2DGroups().to(torch_dtype) + if model_id == "Conv3d": return ModelConv3D().to(torch_dtype) From 481fed7c21de53c214c2cce3dc9661781dfc66d6 Mon Sep 17 00:00:00 2001 From: Georg Slamanig Date: Sat, 1 Mar 2025 13:11:02 +0100 Subject: [PATCH 6/6] fixed failing TC --- tests/test_custom_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_custom_models.py b/tests/test_custom_models.py index ac0f2b34f4..ceccfcbadd 100644 --- a/tests/test_custom_models.py +++ b/tests/test_custom_models.py @@ -908,10 +908,10 @@ def forward(self, X): class ModelConv2DGroups(nn.Module): def __init__(self): super().__init__() - self.conv2d = nn.Conv2d(5, 10, 3, groups=5) + self.conv2d = nn.Conv2d(5, 5, 3, groups=5) self.relu = nn.ReLU() self.flat = nn.Flatten() - self.lin0 = nn.Linear(10, 2) + self.lin0 = nn.Linear(5, 2) self.sm = nn.LogSoftmax(dim=-1) def forward(self, X):