From 8de3b73c0a302590bd1733743cbecf690caf7ccf Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 22 Nov 2023 17:09:26 +0530 Subject: [PATCH 1/2] fix `add_weighted_adapter` method Co-Authored-By: Benjamin Bossan Co-Authored-By: jihuishan <151612440+jihuishan@users.noreply.github.com> --- src/peft/tuners/lora/model.py | 5 +++-- tests/testing_common.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/peft/tuners/lora/model.py b/src/peft/tuners/lora/model.py index a5b7735ce3..653a684276 100644 --- a/src/peft/tuners/lora/model.py +++ b/src/peft/tuners/lora/model.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import math import operator import re import warnings @@ -517,8 +518,8 @@ def add_weighted_adapter( current_adapter_lora_B = target.lora_embedding_B[adapter] else: continue - target_lora_A.data += current_adapter_lora_A.data * weight * target.scaling[adapter] - target_lora_B.data += current_adapter_lora_B.data + target_lora_A.data += current_adapter_lora_A.data * math.sqrt(weight) * target.scaling[adapter] + target_lora_B.data += current_adapter_lora_B.data * math.sqrt(weight) elif combination_type == "cat": loras_A, loras_B = [], [] for adapter, weight in zip(adapters, weights): diff --git a/tests/testing_common.py b/tests/testing_common.py index 17b9f147c2..b57ef006b0 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -163,6 +163,7 @@ class PeftCommonTester: transformers_class (`transformers.PreTrainedModel`): The transformers class that is being tested. """ + torch_device = infer_device() transformers_class = None @@ -1021,6 +1022,14 @@ def _test_weighted_combination_of_adapters(self, model_id, config_cls, config_kw adapter_list[:2], weight_list[:2], "multi_adapter_linear_reweighting", combination_type="linear" ) + # test linear re-weighting with multiple adapters with only single adapter having weight 1.0 + model.add_weighted_adapter( + adapter_list[:2], + [weight_list[0], 0], + "multi_adapter_linear_reweighting_single_enabled", + combination_type="linear", + ) + with self.assertRaises(ValueError): model.add_weighted_adapter( adapter_list[1:], @@ -1034,6 +1043,7 @@ def _test_weighted_combination_of_adapters(self, model_id, config_cls, config_kw "multi_adapter_svd_reweighting", "multi_adapter_cat_reweighting", "multi_adapter_linear_reweighting", + "multi_adapter_linear_reweighting_single_enabled", ] for new_adapter in new_adapters: self.assertTrue(new_adapter in model.peft_config) From 5151cc2de30582fefc22a6d9489f362f87b8f848 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 22 Nov 2023 17:12:18 +0530 Subject: [PATCH 2/2] Update testing_common.py --- tests/testing_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testing_common.py b/tests/testing_common.py index b57ef006b0..00809c2bc1 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -1022,7 +1022,7 @@ def _test_weighted_combination_of_adapters(self, model_id, config_cls, config_kw adapter_list[:2], weight_list[:2], "multi_adapter_linear_reweighting", combination_type="linear" ) - # test linear re-weighting with multiple adapters with only single adapter having weight 1.0 + # test linear re-weighting with multiple adapters with only first adapter having non zero weight model.add_weighted_adapter( adapter_list[:2], [weight_list[0], 0],