🦆 bump version

README.md

@@ -9,7 +9,7 @@ Small Library of Torch Adaptation modules
 - [X] LoRA
 - [X] Prompt Tuning
 - [X] Bottleneck Adapter
-- [ ] Prefix Tuning
+- [X] Prefix Tuning
 - [ ] P-Tuning
 
 # Installation

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "torch-adapters"
-version = "0.0.5"
+version = "0.0.6"
 description = "Small Library of Torch Adaptation modules"
 authors = ["ma2za <mazzapaolo2019@gmail.com>"]
 license = "MIT"

src/torch_adapters/adapters/lora.py

@@ -20,7 +20,7 @@ class LoRA(nn.Linear, AdapterMixin):
     """
 
     def __init__(
-        self, src: nn.Linear, alpha: int = 8, r: int = 8, dropout: float = 0.0
+            self, src: nn.Linear, alpha: int = 8, r: int = 8, dropout: float = 0.0
     ):
         super().__init__(src.in_features, src.out_features)
 
@@ -30,8 +30,8 @@ def __init__(
         self.lora_weight = nn.Sequential(
             OrderedDict(
                 [
-                    ("dropout", nn.Dropout(p=dropout)),
                     ("A", nn.Linear(self.in_features, r, bias=False)),
+                    ("dropout", nn.Dropout(p=dropout)),
                     ("B", nn.Linear(r, self.out_features, bias=False)),
                 ]
             )
@@ -48,13 +48,13 @@ def merge(self) -> nn.Linear:
         # TODO check if matrix transpose is required
         merged_layer = nn.Linear(self.in_features, self.out_features)
         merged_weight = self.weight.data + (self.alpha / self.r) * (
-            self.lora_weight.B.weight.data @ self.lora_weight.A.weight.data
+                self.lora_weight.B.weight.data @ self.lora_weight.A.weight.data
         )
         merged_layer.weight.data = merged_weight.detach().clone().to(self.weight.device)
         merged_layer.bias.data = self.bias.data.detach().clone().to(self.bias.device)
         return merged_layer
 
     def forward(self, input_ids: Tensor) -> Tensor:
         return super().forward(input_ids) + self.lora_weight(input_ids) * (
-            self.alpha / self.r
+                self.alpha / self.r
         )

src/torch_adapters/utils.py

@@ -20,7 +20,7 @@
 
 
 def add_prefix_tuning(
-    model: nn.Module, layers_names: List[str], config: Dict
+        model: nn.Module, layers_names: List[str], config: Dict
 ) -> torch.nn.Module:
     for name, module in model.named_modules():
         if any([i in name for i in layers_names]):
@@ -48,7 +48,7 @@ def drop_prefix_tuning_reparametrization(model: nn.Module):
 
 
 def add_adapter(
-    model: nn.Module, layers_names: List[str], config: Dict
+        model: nn.Module, layers_names: List[str], config: Dict
 ) -> torch.nn.Module:
     """
 
@@ -74,7 +74,7 @@ def add_adapter(
 
 
 def add_lora(
-    model: nn.Module, layers_names: List[str], config: Dict
+        model: nn.Module, layers_names: List[str], config: Dict
 ) -> torch.nn.Module:
     """
 
@@ -100,7 +100,10 @@ def add_lora(
 
             module.__setattr__(
                 attr_name,
-                LoRA(attr, alpha=config.get("alpha", 8), r=config.get("r", 8)),
+                LoRA(attr,
+                     alpha=config.get("alpha", 8),
+                     r=config.get("r", 8),
+                     dropout=config.get("dropout", 0.0)),
             )
     return model