Fix FLUX diffusers LoRA models with no .proj_mlp layers

invokeai/backend/lora/conversions/flux_diffusers_lora_conversion_utils.py

@@ -45,8 +45,9 @@ def lora_model_from_flux_diffusers_state_dict(state_dict: Dict[str, torch.Tensor
     # Constants for FLUX.1
     num_double_layers = 19
     num_single_layers = 38
-    # inner_dim = 3072
-    # mlp_ratio = 4.0
+    hidden_size = 3072
+    mlp_ratio = 4.0
+    mlp_hidden_dim = int(hidden_size * mlp_ratio)
 
     layers: dict[str, AnyLoRALayer] = {}
 
@@ -62,30 +63,43 @@ def add_lora_layer_if_present(src_key: str, dst_key: str) -> None:
             layers[dst_key] = LoRALayer.from_state_dict_values(values=value)
             assert len(src_layer_dict) == 0
 
-    def add_qkv_lora_layer_if_present(src_keys: list[str], dst_qkv_key: str) -> None:
+    def add_qkv_lora_layer_if_present(
+        src_keys: list[str],
+        src_weight_shapes: list[tuple[int, int]],
+        dst_qkv_key: str,
+        allow_missing_keys: bool = False,
+    ) -> None:
         """Handle the Q, K, V matrices for a transformer block. We need special handling because the diffusers format
         stores them in separate matrices, whereas the BFL format used internally by InvokeAI concatenates them.
         """
-        # We expect that either all src keys are present or none of them are. Verify this.
-        keys_present = [key in grouped_state_dict for key in src_keys]
-        assert all(keys_present) or not any(keys_present)
-
         # If none of the keys are present, return early.
+        keys_present = [key in grouped_state_dict for key in src_keys]
         if not any(keys_present):
             return
 
-        src_layer_dicts = [grouped_state_dict.pop(key) for key in src_keys]
         sub_layers: list[LoRALayer] = []
-        for src_layer_dict in src_layer_dicts:
-            values = {
-                "lora_down.weight": src_layer_dict.pop("lora_A.weight"),
-                "lora_up.weight": src_layer_dict.pop("lora_B.weight"),
-            }
-            if alpha is not None:
-                values["alpha"] = torch.tensor(alpha)
-            sub_layers.append(LoRALayer.from_state_dict_values(values=values))
-            assert len(src_layer_dict) == 0
-        layers[dst_qkv_key] = ConcatenatedLoRALayer(lora_layers=sub_layers, concat_axis=0)
+        for src_key, src_weight_shape in zip(src_keys, src_weight_shapes, strict=True):
+            src_layer_dict = grouped_state_dict.pop(src_key, None)
+            if src_layer_dict is not None:
+                values = {
+                    "lora_down.weight": src_layer_dict.pop("lora_A.weight"),
+                    "lora_up.weight": src_layer_dict.pop("lora_B.weight"),
+                }
+                if alpha is not None:
+                    values["alpha"] = torch.tensor(alpha)
+                assert values["lora_down.weight"].shape[1] == src_weight_shape[1]
+                assert values["lora_up.weight"].shape[0] == src_weight_shape[0]
+                sub_layers.append(LoRALayer.from_state_dict_values(values=values))
+                assert len(src_layer_dict) == 0
+            else:
+                if not allow_missing_keys:
+                    raise ValueError(f"Missing LoRA layer: '{src_key}'.")
+                values = {
+                    "lora_up.weight": torch.zeros((src_weight_shape[0], 1)),
+                    "lora_down.weight": torch.zeros((1, src_weight_shape[1])),
+                }
+                sub_layers.append(LoRALayer.from_state_dict_values(values=values))
+        layers[dst_qkv_key] = ConcatenatedLoRALayer(lora_layers=sub_layers)
 
     # time_text_embed.timestep_embedder -> time_in.
     add_lora_layer_if_present("time_text_embed.timestep_embedder.linear_1", "time_in.in_layer")
@@ -118,6 +132,7 @@ def add_qkv_lora_layer_if_present(src_keys: list[str], dst_qkv_key: str) -> None
                 f"transformer_blocks.{i}.attn.to_k",
                 f"transformer_blocks.{i}.attn.to_v",
             ],
+            [(hidden_size, hidden_size), (hidden_size, hidden_size), (hidden_size, hidden_size)],
             f"double_blocks.{i}.img_attn.qkv",
         )
         add_qkv_lora_layer_if_present(
@@ -126,6 +141,7 @@ def add_qkv_lora_layer_if_present(src_keys: list[str], dst_qkv_key: str) -> None
                 f"transformer_blocks.{i}.attn.add_k_proj",
                 f"transformer_blocks.{i}.attn.add_v_proj",
             ],
+            [(hidden_size, hidden_size), (hidden_size, hidden_size), (hidden_size, hidden_size)],
             f"double_blocks.{i}.txt_attn.qkv",
         )
 
@@ -175,7 +191,14 @@ def add_qkv_lora_layer_if_present(src_keys: list[str], dst_qkv_key: str) -> None
                 f"single_transformer_blocks.{i}.attn.to_v",
                 f"single_transformer_blocks.{i}.proj_mlp",
             ],
+            [
+                (hidden_size, hidden_size),
+                (hidden_size, hidden_size),
+                (hidden_size, hidden_size),
+                (mlp_hidden_dim, hidden_size),
+            ],
             f"single_blocks.{i}.linear1",
+            allow_missing_keys=True,
         )
 
         # Output projections.