Update OFT to fix merge bugs

src/peft/config.py

@@ -216,17 +216,11 @@ def _get_peft_type(
 
     @classmethod
     def check_kwargs(cls, **kwargs):
-        r"""
-        Check if the kwargs are valid for the configuration.
+        """Check kwargs before initializing the config instance.
+
+        Subclasses can override this method to add specific checks.
 
-        Args:
-            kwargs (additional keyword arguments, *optional*):
-                Additional keyword arguments passed along to the child class initialization.
         """
-        if "oft_block_size" in kwargs:
-            warnings.warn(
-                'OFT has been updated since 0.12.1.dev0. Your trained adapter weights may not be compatible with the latest version of OFT. Please retrain your adapter weights.')
-            )
         return kwargs
 
     @property

src/peft/tuners/oft/config.py

@@ -18,6 +18,8 @@
 from peft.config import PeftConfig
 from peft.utils import PeftType
 
+import warnings
+
 
 @dataclass
 class OFTConfig(PeftConfig):
@@ -162,3 +164,20 @@ def __post_init__(self):
             raise ValueError(
                 f"You can only specify either r ({self.r}) or oft_block_size ({self.oft_block_size}), but not both simultaneously, because r x oft_block_size == in_features."
             )
+
+    @classmethod
+    def check_kwargs(cls, **kwargs):
+        r"""
+        Check if the kwargs are valid for the configuration.
+
+        Args:
+            kwargs (additional keyword arguments, *optional*):
+                Additional keyword arguments passed along to the child class initialization.
+        """
+        if "oft_block_size" not in kwargs:
+            raise ValueError(
+                'OFT has been updated since PEFT 0.13.0. Your trained adapter weights is incompatible with the latest version of OFT. Please retrain your adapter weights with newer PEFT versions.'
+                'Downgrade PEFT to version 0.12.0 to merge the old adapter weights.'
+            )
+        return super().check_kwargs(**kwargs)
+

src/peft/tuners/oft/layer.py

@@ -169,12 +169,16 @@ def update_layer(
         self.oft_dropout.update(nn.ModuleDict({adapter_name: oft_dropout_layer}))
 
         if r == 0 and oft_block_size != 0:
-            if self.in_features % oft_block_size != 0:
-                raise ValueError(f"Input features ({self.in_features}) should be divisible by `oft_block_size` ({oft_block_size})")
+            if self.in_features % oft_block_size != 0 or oft_block_size > self.in_features:
+                warnings.warn(f"Invalid `oft_block_size` ({oft_block_size})!")
+                oft_block_size = self.adjust_oft_parameters(self.in_features, oft_block_size)
+                warnings.warn(f"Adjusted `oft_block_size` to ({oft_block_size}).")
             r = int(self.in_features // oft_block_size)
         elif r != 0 and oft_block_size == 0:
-            if self.in_features % r != 0:
-                raise ValueError(f"Input features ({self.in_features}) should be divisible by `r` ({r})!")
+            if self.in_features % r != 0 or r > self.in_features:
+                warnings.warn(f"Invalid `r` ({r})!")
+                r = self.adjust_oft_parameters(self.in_features, r)
+                warnings.warn(f"Adjusted `r` to ({r}).")
             oft_block_size = int(self.in_features // r)
         else:
             raise ValueError("Something went wrong, please report this error: https://github.com/huggingface/peft/issues")
@@ -263,6 +267,26 @@ def _project_batch(self, oft_r, eps=1e-5):
         out = torch.where(mask, oft_r, I + eps * (diff / norm_diff))
         return out
 
+    def adjust_oft_parameters(self, in_features, params):
+        """
+        Adjust the OFT parameters to be divisible by the in_features dimension.
+        """
+        if params < in_features:
+            higher_params = params
+            while higher_params <= in_features and in_features % higher_params != 0:
+                higher_params += 1
+        else:
+            return in_features
+
+        lower_params = params
+        while lower_params > 1 and in_features % lower_params != 0:
+            lower_params -= 1
+
+        if (params - lower_params) <= (higher_params - params):
+            return lower_params
+        else:
+            return higher_params
+
 
 class Linear(nn.Module, OFTLayer):
     """OFT implemented in Linear layer"""
@@ -490,16 +514,16 @@ def update_layer(
         conv_filter_dim = self.in_features * base_layer.kernel_size[0] * base_layer.kernel_size[0]
 
         if r == 0 and oft_block_size != 0:
-            if conv_filter_dim % oft_block_size != 0:
-                raise ValueError(
-                    f"Convolutional kernel dimension ({conv_filter_dim}) must be divisible by conv_filter_dim ({conv_filter_dim})!"
-                )
+            if conv_filter_dim % oft_block_size != 0 or oft_block_size > conv_filter_dim:
+                warnings.warn(f"Invalid `oft_block_size` ({oft_block_size})!")
+                oft_block_size = self.adjust_oft_parameters(conv_filter_dim, oft_block_size)
+                warnings.warn(f"Adjusted `oft_block_size` to ({oft_block_size}).")
             r = int(conv_filter_dim // oft_block_size)
         elif r != 0 and oft_block_size == 0:
-            if conv_filter_dim % r != 0:
-                raise ValueError(
-                    f"Convolutional kernel dimension ({conv_filter_dim}) must be divisible by r ({r})!"
-                )
+            if conv_filter_dim % r != 0 or r > conv_filter_dim:
+                warnings.warn(f"Invalid `r` ({r})!")
+                r = self.adjust_oft_parameters(conv_filter_dim, r)
+                warnings.warn(f"Adjusted `r` to ({r}).")
             oft_block_size = int(conv_filter_dim // r)
         else:
             raise ValueError("Something went wrong, please report this error: https://github.com/huggingface/peft/issues")