'NoneType' object has no attribute 'image_projection_layers'

[irarayzelji2002]

Is this an installation problem of IP Adapter or is my code incorrect somewhere?

Where I initialized IP Adapter

def modify_weights(weights_path):
    try:
        state_dict = torch.load(weights_path, map_location="cuda:0")
    except Exception as e:
        print(f"Error loading weights: {e}")
        return {}
    modified_state_dict = {}
    for key, value in state_dict.items():
        if isinstance(value, torch.Tensor):
            print(f"Original shape of {key}: {value.shape}")
            # Handling decreased size (e.g., from 1024 to 768)
            if value.shape == torch.Size([320, 768]):
                factor = 1024 / 768  # Multiply by this factor to match expected shape
                modified_state_dict[key] = value * factor
            # Handling increased size (e.g., from 320x1024 to 640x2048)
            elif value.shape == torch.Size([640, 2048]):
                factor = 2  # Divide by this factor to reduce the size
                modified_state_dict[key] = value / factor
            else:
                modified_state_dict[key] = value  # No change for non-mismatched shapes
            print(f"Modified shape of {key}: {modified_state_dict[key].shape}")
    return modified_state_dict

def setup_pipeline():
pipeline_text2image = AutoPipelineForText2Image.from_pretrained(
    "stabilityai/stable-diffusion-2-1-base", torch_dtype=torch.float16, variant="fp16", use_safetensors=True,
).to("cuda")
pipeline_image2image = AutoPipelineForImage2Image.from_pipe(pipeline_text2image).to("cuda")
# Load the IP Adapter weights manually, Modify the specific weights
ip_adapter_path = r"C:\Users\Ira\.cache\huggingface\hub\models--h94--IP-Adapter\snapshots\018e402774aeeddd60609b4ecdb7e298259dc729\sdxl_models\ip-adapter-plus_sdxl_vit-h.bin"
modified_weights = modify_weights(ip_adapter_path, factor=2)
pipeline_image2image.unet.load_state_dict(modified_weights, strict=False)
pipeline_image2image.set_ip_adapter_scale(0.6)
# pipeline_image2image.unet.encoder_hid_proj = None
torch_generator = torch.Generator(device="cpu").manual_seed(4)
device = torch.device("cuda")

Code that throwed an exception

new_images = pipeline_image2image(
    prompt = "best quality, high quality", 
    image = generated_image,
    ip_adapter_image = style_reference_loaded,
    ip_adapter_image_embeds = None,
    negative_prompt = negative_prompt,
    generator = torch_generator,
    strength = 0.6,
    height = 512, width = 512
).images

Testing Pipelines
Code to test

def test_pipelines():
    if pipeline_text2image is None or pipeline_image2image is None or torch_generator is None:
        if pipeline_text2image is None:
            print("pipeline_text2image not loaded correctly.")
        if pipeline_image2image is None:
            print("pipeline_image2image not loaded correctly.")
        if torch_generator is None:
            print("torch_generator not loaded correctly.")
    else:
        print(f"========pipeline_image2image attributes: {dir(pipeline_image2image)}")
        if hasattr(pipeline_image2image, 'unet'):
            print(f"========unet attributes: {dir(pipeline_image2image.unet)}")
            if hasattr(pipeline_image2image.unet, 'encoder_hid_proj'):
                print(f"========encoder_hid_proj attributes: {dir(pipeline_image2image.unet.encoder_hid_proj)}")
        print("Pipelines loaded successfully.")

Logs showed

========Model loaded successfully. Running on device: cuda========
========pipeline_image2image attributes: ['__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_best_guess_weight_name', '_callback_tensor_inputs', '_check_text_inv_inputs', '_deprecated_kwargs', '_dict_from_json_file', '_encode_prompt', '_exclude_from_cpu_offload', '_execution_device', '_extend_tokens_and_embeddings', '_fetch_state_dict', '_get_init_keys', '_get_signature_keys', '_get_signature_types', '_internal_dict', '_is_onnx', '_load_connected_pipes', '_lora_loadable_modules', '_maybe_convert_prompt', '_optional_components', '_optionally_disable_offloading', '_retrieve_tokens_and_embeddings', '_upload_folder', 'check_inputs', 'clip_skip', 'components', 'config', 'config_name', 'cross_attention_kwargs', 'decode_latents', 'delete_adapters', 'device', 'disable_attention_slicing', 'disable_freeu', 'disable_lora', 'disable_vae_slicing', 'disable_vae_tiling', 'disable_xformers_memory_efficient_attention', 'do_classifier_free_guidance', 'download', 'dtype', 'enable_attention_slicing', 'enable_freeu', 'enable_lora', 'enable_model_cpu_offload', 'enable_sequential_cpu_offload', 'enable_vae_slicing', 'enable_vae_tiling', 'enable_xformers_memory_efficient_attention', 'encode_image', 'encode_prompt', 'extract_init_dict', 'feature_extractor', 'from_config', 'from_pipe', 'from_pretrained', 'from_single_file', 'fuse_lora', 'fuse_qkv_projections', 'get_active_adapters', 'get_config_dict', 'get_guidance_scale_embedding', 'get_list_adapters', 'get_timesteps', 'guidance_scale', 'has_compatibles', 'hf_device_map', 'ignore_for_config', 'image_encoder', 'image_processor', 'interrupt', 'load_config', 'load_ip_adapter', 'load_lora_into_text_encoder', 'load_lora_into_unet', 'load_lora_weights', 'load_textual_inversion', 'lora_scale', 'lora_state_dict', 'maybe_convert_prompt', 'maybe_free_model_hooks', 'model_cpu_offload_seq', 'name_or_path', 'num_fused_loras', 'num_timesteps', 'numpy_to_pil', 'pack_weights', 'prepare_extra_step_kwargs', 'prepare_ip_adapter_image_embeds', 'prepare_latents', 'progress_bar', 'push_to_hub', 'register_modules', 'register_to_config', 'remove_all_hooks', 'reset_device_map', 'run_safety_checker', 'safety_checker', 'save_config', 'save_lora_weights', 'save_pretrained', 'scheduler', 'set_adapters', 'set_attention_slice', 'set_ip_adapter_scale', 'set_lora_device', 'set_progress_bar_config', 'set_use_memory_efficient_attention_xformers', 'text_encoder', 'text_encoder_name', 'to', 'to_json_file', 'to_json_string', 'tokenizer', 'unet', 'unet_name', 'unfuse_lora', 'unfuse_qkv_projections', 'unload_ip_adapter', 'unload_lora_weights', 'unload_textual_inversion', 'vae', 'vae_scale_factor', 'write_lora_layers']
========unet attributes: ['T_destination', '__annotations__', '__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_apply', '_automatically_saved_args', '_backward_hooks', '_backward_pre_hooks', '_buffers', '_call_impl', '_check_config', '_compiled_call_impl', '_convert_deprecated_attention_blocks', '_convert_ip_adapter_attn_to_diffusers', '_convert_ip_adapter_image_proj_to_diffusers', '_deprecated_kwargs', '_dict_from_json_file', '_forward_hooks', '_forward_hooks_always_called', '_forward_hooks_with_kwargs', '_forward_pre_hooks', '_forward_pre_hooks_with_kwargs', '_fuse_lora_apply', '_get_backward_hooks', '_get_backward_pre_hooks', '_get_custom_diffusion_state_dict', '_get_init_keys', '_get_name', '_get_no_split_modules', '_get_signature_keys', '_hf_peft_config_loaded', '_internal_dict', '_is_full_backward_hook', '_keys_to_ignore_on_load_unexpected', '_load_from_state_dict', '_load_ip_adapter_loras', '_load_ip_adapter_weights', '_load_pretrained_model', '_load_state_dict_post_hooks', '_load_state_dict_pre_hooks', '_maybe_warn_non_full_backward_hook', '_modules', '_named_members', '_no_split_modules', '_non_persistent_buffers_set', '_optionally_disable_offloading', '_parameters', '_process_custom_diffusion', '_process_lora', '_register_load_state_dict_pre_hook', '_register_state_dict_hook', '_replicate_for_data_parallel', '_save_to_state_dict', '_set_add_embedding', '_set_class_embedding', '_set_encoder_hid_proj', '_set_gradient_checkpointing', '_set_pos_net_if_use_gligen', '_set_time_proj', '_slow_forward', '_state_dict_hooks', '_state_dict_pre_hooks', '_supports_gradient_checkpointing', '_temp_convert_self_to_deprecated_attention_blocks', '_tome_info', '_undo_temp_convert_self_to_deprecated_attention_blocks', '_unfuse_lora_apply', '_upload_folder', '_version', '_wrapped_call_impl', 'active_adapters', 'add_adapter', 'add_module', 'apply', 'attn_processors', 'bfloat16', 'buffers', 'call_super_init', 'children', 'class_embedding', 'compile', 'config', 'config_name', 'conv_act', 'conv_in', 'conv_norm_out', 'conv_out', 'cpu', 'cuda', 'delete_adapters', 'device', 'disable_adapters', 'disable_freeu', 'disable_gradient_checkpointing', 'disable_lora', 'disable_npu_flash_attention', 'disable_xformers_memory_efficient_attention', 'double', 'down_blocks', 'dtype', 'dump_patches', 'enable_adapters', 'enable_freeu', 'enable_gradient_checkpointing', 'enable_lora', 'enable_npu_flash_attention', 'enable_xformers_memory_efficient_attention', 'encoder_hid_proj', 'eval', 'extra_repr', 'extract_init_dict', 'float', 'forward', 'from_config', 'from_pretrained', 'from_single_file', 'fuse_lora', 'fuse_qkv_projections', 'get_aug_embed', 'get_buffer', 'get_class_embed', 'get_config_dict', 'get_extra_state', 'get_parameter', 'get_submodule', 'get_time_embed', 'half', 'has_compatibles', 'ignore_for_config', 'ipu', 'is_gradient_checkpointing', 'load_attn_procs', 'load_config', 'load_state_dict', 'mid_block', 'modules', 'named_buffers', 'named_children', 'named_modules', 'named_parameters', 'num_parameters', 'num_upsamplers', 'parameters', 'process_encoder_hidden_states', 'push_to_hub', 'register_backward_hook', 'register_buffer', 'register_forward_hook', 'register_forward_pre_hook', 'register_full_backward_hook', 'register_full_backward_pre_hook', 'register_load_state_dict_post_hook', 'register_module', 'register_parameter', 'register_state_dict_pre_hook', 'register_to_config', 'requires_grad_', 'sample_size', 'save_attn_procs', 'save_config', 'save_pretrained', 'set_adapter', 'set_adapters', 'set_attention_slice', 'set_attn_processor', 'set_default_attn_processor', 'set_extra_state', 'set_use_memory_efficient_attention_xformers', 'set_use_npu_flash_attention', 'share_memory', 'state_dict', 'text_encoder_name', 'time_embed_act', 'time_embedding', 'time_proj', 'to', 'to_empty', 'to_json_file', 'to_json_string', 'train', 'training', 'type', 'unet_name', 'unfuse_lora', 'unfuse_qkv_projections', 'unload_lora', 'up_blocks', 'xpu', 'zero_grad']
========encoder_hid_proj attributes: ['__bool__', '__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__']
Pipelines loaded successfully.

Other Notes
I don't think this is related but before the error/exception there is this warning:

C:\Users\Ira\AppData\Roaming\Python\Python312\site-packages\transformers\models\clip\modeling_clip.py:480: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at ..\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:455.)  
  attn_output = torch.nn.functional.scaled_dot_product_attention(

After that is this then it stops:
'NoneType' object has no attribute 'image_projection_layers'

I tried SD1.4 with IP Adapter 1.5 and it has more attributes especially under encoder_hid_proj and didn't show the error of NoneType object has no attribute 'image_projection_layers'
Here is the full log of working one, but I wish to use SD2.1 only, not SDXL because my computer can't handle it (always memory full on just a single image generation, and can't save the image after processing for minutes).

========pipeline_image2image attributes: ['__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_best_guess_weight_name', '_callback_tensor_inputs', '_check_text_inv_inputs', '_deprecated_kwargs', '_dict_from_json_file', '_encode_prompt', '_exclude_from_cpu_offload', '_execution_device', '_extend_tokens_and_embeddings', '_fetch_state_dict', '_get_init_keys', '_get_signature_keys', '_get_signature_types', '_internal_dict', '_is_onnx', '_load_connected_pipes', '_lora_loadable_modules', '_maybe_convert_prompt', '_optional_components', '_optionally_disable_offloading', '_retrieve_tokens_and_embeddings', '_upload_folder', 'check_inputs', 'clip_skip', 'components', 'config', 'config_name', 'cross_attention_kwargs', 'decode_latents', 'delete_adapters', 'device', 'disable_attention_slicing', 'disable_freeu', 'disable_lora', 'disable_vae_slicing', 'disable_vae_tiling', 'disable_xformers_memory_efficient_attention', 'do_classifier_free_guidance', 'download', 'dtype', 'enable_attention_slicing', 'enable_freeu', 'enable_lora', 'enable_model_cpu_offload', 'enable_sequential_cpu_offload', 'enable_vae_slicing', 'enable_vae_tiling', 'enable_xformers_memory_efficient_attention', 'encode_image', 'encode_prompt', 'extract_init_dict', 'feature_extractor', 'from_config', 'from_pipe', 'from_pretrained', 'from_single_file', 'fuse_lora', 'fuse_qkv_projections', 'get_active_adapters', 'get_config_dict', 'get_guidance_scale_embedding', 'get_list_adapters', 'get_timesteps', 'guidance_scale', 'has_compatibles', 'hf_device_map', 'ignore_for_config', 'image_encoder', 'image_processor', 'interrupt', 'load_config', 'load_ip_adapter', 'load_lora_into_text_encoder', 'load_lora_into_unet', 'load_lora_weights', 'load_textual_inversion', 'lora_scale', 'lora_state_dict', 'maybe_convert_prompt', 'maybe_free_model_hooks', 'model_cpu_offload_seq', 'name_or_path', 'num_fused_loras', 'num_timesteps', 'numpy_to_pil', 'pack_weights', 'prepare_extra_step_kwargs', 'prepare_ip_adapter_image_embeds', 'prepare_latents', 'progress_bar', 'push_to_hub', 'register_modules', 'register_to_config', 'remove_all_hooks', 'reset_device_map', 'run_safety_checker', 'safety_checker', 'save_config', 'save_lora_weights', 'save_pretrained', 'scheduler', 'set_adapters', 'set_attention_slice', 'set_ip_adapter_scale', 'set_lora_device', 'set_progress_bar_config', 'set_use_memory_efficient_attention_xformers', 'text_encoder', 'text_encoder_name', 'to', 'to_json_file', 'to_json_string', 'tokenizer', 'unet', 'unet_name', 'unfuse_lora', 'unfuse_qkv_projections', 'unload_ip_adapter', 'unload_lora_weights', 'unload_textual_inversion', 'vae', 'vae_scale_factor', 'write_lora_layers']
========unet attributes: ['T_destination', '__annotations__', '__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_apply', '_automatically_saved_args', '_backward_hooks', '_backward_pre_hooks', '_buffers', '_call_impl', '_check_config', '_compiled_call_impl', '_convert_deprecated_attention_blocks', '_convert_ip_adapter_attn_to_diffusers', '_convert_ip_adapter_image_proj_to_diffusers', '_deprecated_kwargs', '_dict_from_json_file', '_forward_hooks', '_forward_hooks_always_called', '_forward_hooks_with_kwargs', '_forward_pre_hooks', '_forward_pre_hooks_with_kwargs', '_fuse_lora_apply', '_get_backward_hooks', '_get_backward_pre_hooks', '_get_custom_diffusion_state_dict', '_get_init_keys', '_get_name', '_get_no_split_modules', '_get_signature_keys', '_hf_peft_config_loaded', '_internal_dict', '_is_full_backward_hook', '_keys_to_ignore_on_load_unexpected', '_load_from_state_dict', '_load_ip_adapter_loras', '_load_ip_adapter_weights', '_load_pretrained_model', '_load_state_dict_post_hooks', '_load_state_dict_pre_hooks', '_maybe_warn_non_full_backward_hook', '_modules', '_named_members', '_no_split_modules', '_non_persistent_buffers_set', '_optionally_disable_offloading', '_parameters', '_process_custom_diffusion', '_process_lora', '_register_load_state_dict_pre_hook', '_register_state_dict_hook', '_replicate_for_data_parallel', '_save_to_state_dict', '_set_add_embedding', '_set_class_embedding', '_set_encoder_hid_proj', '_set_gradient_checkpointing', '_set_pos_net_if_use_gligen', '_set_time_proj', '_slow_forward', '_state_dict_hooks', '_state_dict_pre_hooks', '_supports_gradient_checkpointing', '_temp_convert_self_to_deprecated_attention_blocks', '_tome_info', '_undo_temp_convert_self_to_deprecated_attention_blocks', '_unfuse_lora_apply', '_upload_folder', '_version', '_wrapped_call_impl', 'active_adapters', 'add_adapter', 'add_module', 'apply', 'attn_processors', 'bfloat16', 'buffers', 'call_super_init', 'children', 'class_embedding', 'compile', 'config', 'config_name', 'conv_act', 'conv_in', 'conv_norm_out', 'conv_out', 'cpu', 'cuda', 'delete_adapters', 'device', 'disable_adapters', 'disable_freeu', 'disable_gradient_checkpointing', 'disable_lora', 'disable_npu_flash_attention', 'disable_xformers_memory_efficient_attention', 'double', 'down_blocks', 'dtype', 'dump_patches', 'enable_adapters', 'enable_freeu', 'enable_gradient_checkpointing', 'enable_lora', 'enable_npu_flash_attention', 'enable_xformers_memory_efficient_attention', 'encoder_hid_proj', 'eval', 'extra_repr', 'extract_init_dict', 'float', 'forward', 'from_config', 'from_pretrained', 'from_single_file', 'fuse_lora', 'fuse_qkv_projections', 'get_aug_embed', 'get_buffer', 'get_class_embed', 'get_config_dict', 'get_extra_state', 'get_parameter', 'get_submodule', 'get_time_embed', 'half', 'has_compatibles', 'ignore_for_config', 'ipu', 'is_gradient_checkpointing', 'load_attn_procs', 'load_config', 'load_state_dict', 'mid_block', 'modules', 'named_buffers', 'named_children', 'named_modules', 'named_parameters', 'num_parameters', 'num_upsamplers', 'parameters', 'process_encoder_hidden_states', 'push_to_hub', 'register_backward_hook', 'register_buffer', 'register_forward_hook', 'register_forward_pre_hook', 'register_full_backward_hook', 'register_full_backward_pre_hook', 'register_load_state_dict_post_hook', 'register_module', 'register_parameter', 'register_state_dict_pre_hook', 'register_to_config', 'requires_grad_', 'sample_size', 'save_attn_procs', 'save_config', 'save_pretrained', 'set_adapter', 'set_adapters', 'set_attention_slice', 'set_attn_processor', 'set_default_attn_processor', 'set_extra_state', 'set_use_memory_efficient_attention_xformers', 'set_use_npu_flash_attention', 'share_memory', 'state_dict', 'text_encoder_name', 'time_embed_act', 'time_embedding', 'time_proj', 'to', 'to_empty', 'to_json_file', 'to_json_string', 'train', 'training', 'type', 'unet_name', 'unfuse_lora', 'unfuse_qkv_projections', 'unload_lora', 'up_blocks', 'xpu', 'zero_grad']
========encoder_hid_proj attributes: ['T_destination', '__annotations__', '__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_apply', '_backward_hooks', '_backward_pre_hooks', '_buffers', '_call_impl', '_compiled_call_impl', '_forward_hooks', '_forward_hooks_always_called', '_forward_hooks_with_kwargs', '_forward_pre_hooks', '_forward_pre_hooks_with_kwargs', '_get_backward_hooks', '_get_backward_pre_hooks', '_get_name', '_is_full_backward_hook', '_load_from_state_dict', '_load_state_dict_post_hooks', '_load_state_dict_pre_hooks', '_maybe_warn_non_full_backward_hook', '_modules', '_named_members', '_non_persistent_buffers_set', '_parameters', '_register_load_state_dict_pre_hook', '_register_state_dict_hook', '_replicate_for_data_parallel', '_save_to_state_dict', '_slow_forward', '_state_dict_hooks', '_state_dict_pre_hooks', '_version', '_wrapped_call_impl', 'add_module', 'apply', 'bfloat16', 'buffers', 'call_super_init', 'children', 'compile', 'cpu', 'cuda', 'double', 'dump_patches', 'eval', 'extra_repr', 'float', 'forward', 'get_buffer', 'get_extra_state', 'get_parameter', 'get_submodule', 'half', 'image_projection_layers', 'ipu', 'load_state_dict', 'modules', 'named_buffers', 'named_children', 'named_modules', 'named_parameters', 'parameters', 'register_backward_hook', 'register_buffer', 'register_forward_hook', 'register_forward_pre_hook', 'register_full_backward_hook', 'register_full_backward_pre_hook', 'register_load_state_dict_post_hook', 'register_module', 'register_parameter', 'register_state_dict_pre_hook', 'requires_grad_', 'set_extra_state', 'share_memory', 'state_dict', 'to', 'to_empty', 'train', 'training', 'type', 'xpu', 'zero_grad']