Fix save pretrained for granite speech

Author: alex-jw-brooks

src/transformers/models/granite_speech/configuration_granite_speech.py

@@ -38,7 +38,7 @@ def __init__(
 
 ## adapted from transformers.models.blip.configuration_blip_2.Blip2VisionConfig
 class GraniteSpeechProjectorConfig(PretrainedConfig):
-    model_type = "blip_2_qformer"
+    model_type = "granite_speech_qformer"
 
     def __init__(
         self,
@@ -107,9 +107,7 @@ def __init__(
             text_config = CONFIG_MAPPING["granite"]()
 
         if isinstance(projector_config, dict):
-            # TODO - Make this generic after blip2qformer is moved out to its own model dir.
-            if projector_config["model_type"] != "blip_2_qformer":
-                raise ValueError("Granite speech currently requires blip2 qformer as its encoder!")
+            # TODO - In the future, we should make this generic.
             projector_config = GraniteSpeechProjectorConfig(**projector_config)
         elif projector_config is None:
             projector_config = GraniteSpeechProjectorConfig()

src/transformers/models/granite_speech/modeling_granite_speech.py

@@ -1377,6 +1377,18 @@ def generate(self, *args, **kwargs):
                 self.disable_adapters()
         return super().generate(*args, input_features=input_features, **kwargs)
 
+    def save_pretrained(self, *args, **kwargs):
+        # overwrite save_pretrained to first save the adapter if we have one
+        # NOTE - this will use the base model path we are exporting in the lora
+        # adapter, which may not necessarily be the best behavior, but for now
+        # we keep this for portability, since using the local dir causes problems
+        # if the model is loaded from outside of the current working dir.
+        if is_peft_available and self._hf_peft_config_loaded:
+            super().save_pretrained(*args, **kwargs)
+        # Then save the base model afterwards
+        self._hf_peft_config_loaded = False
+        super().save_pretrained(*args, **kwargs)
+
 
 __all__ = [
     "GraniteSpeechForConditionalGeneration",

tests/models/granite_speech/test_modeling_granite_speech.py

@@ -96,7 +96,7 @@ def __init__(
             "layer_norm_eps": 1e-12,
             "llm_dim": 32,
             "max_position_embeddings": 2048,
-            "model_type": "blip_2_qformer",
+            "model_type": "granite_speech_qformer",
             "num_attention_heads": 4,
             "num_hidden_layers": 2,
             "position_embedding_type": "absolute",