diff --git a/docs/source/en/using-diffusers/push_to_hub.md b/docs/source/en/using-diffusers/push_to_hub.md index 58598c3bc443..e54062417dfd 100644 --- a/docs/source/en/using-diffusers/push_to_hub.md +++ b/docs/source/en/using-diffusers/push_to_hub.md @@ -174,10 +174,4 @@ Set `private=True` in the [`~diffusers.utils.PushToHubMixin.push_to_hub`] functi controlnet.push_to_hub("my-controlnet-model-private", private=True) ``` -Private repositories are only visible to you, and other users won't be able to clone the repository and your repository won't appear in search results. Even if a user has the URL to your private repository, they'll receive a `404 - Sorry, we can't find the page you are looking for.` - -To load a model, scheduler, or pipeline from private or gated repositories, set `use_auth_token=True`: - -```py -model = ControlNetModel.from_pretrained("your-namespace/my-controlnet-model-private", use_auth_token=True) -``` +Private repositories are only visible to you, and other users won't be able to clone the repository and your repository won't appear in search results. Even if a user has the URL to your private repository, they'll receive a `404 - Sorry, we can't find the page you are looking for`. You must be [logged in](https://huggingface.co/docs/huggingface_hub/quick-start#login) to load a model from a private repository. \ No newline at end of file diff --git a/examples/community/README.md b/examples/community/README.md index 98780edeccf7..1d13e2822b77 100755 --- a/examples/community/README.md +++ b/examples/community/README.md @@ -512,7 +512,6 @@ device = torch.device('cpu' if not has_cuda else 'cuda') pipe = DiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", safety_checker=None, - use_auth_token=True, custom_pipeline="imagic_stable_diffusion", scheduler = DDIMScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", clip_sample=False, set_alpha_to_one=False) ).to(device) @@ -552,7 +551,6 @@ device = th.device('cpu' if not has_cuda else 'cuda') pipe = DiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", - use_auth_token=True, custom_pipeline="seed_resize_stable_diffusion" ).to(device) @@ -588,7 +586,6 @@ generator = th.Generator("cuda").manual_seed(0) pipe = DiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", - use_auth_token=True, custom_pipeline="/home/mark/open_source/diffusers/examples/community/" ).to(device) @@ -607,7 +604,6 @@ image.save('./seed_resize/seed_resize_{w}_{h}_image.png'.format(w=width, h=heigh pipe_compare = DiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", - use_auth_token=True, custom_pipeline="/home/mark/open_source/diffusers/examples/community/" ).to(device) diff --git a/examples/community/checkpoint_merger.py b/examples/community/checkpoint_merger.py index 10381020bf63..218ac87fe5f1 100644 --- a/examples/community/checkpoint_merger.py +++ b/examples/community/checkpoint_merger.py @@ -5,10 +5,11 @@ import safetensors.torch import torch from huggingface_hub import snapshot_download +from huggingface_hub.utils import validate_hf_hub_args from diffusers import DiffusionPipeline, __version__ from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME -from diffusers.utils import CONFIG_NAME, DIFFUSERS_CACHE, ONNX_WEIGHTS_NAME, WEIGHTS_NAME +from diffusers.utils import CONFIG_NAME, ONNX_WEIGHTS_NAME, WEIGHTS_NAME class CheckpointMergerPipeline(DiffusionPipeline): @@ -57,6 +58,7 @@ def _remove_meta_keys(self, config_dict: Dict): return (temp_dict, meta_keys) @torch.no_grad() + @validate_hf_hub_args def merge(self, pretrained_model_name_or_path_list: List[Union[str, os.PathLike]], **kwargs): """ Returns a new pipeline object of the class 'DiffusionPipeline' with the merged checkpoints(weights) of the models passed @@ -69,7 +71,7 @@ def merge(self, pretrained_model_name_or_path_list: List[Union[str, os.PathLike] **kwargs: Supports all the default DiffusionPipeline.get_config_dict kwargs viz.. - cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map. + cache_dir, resume_download, force_download, proxies, local_files_only, token, revision, torch_dtype, device_map. alpha - The interpolation parameter. Ranges from 0 to 1. It affects the ratio in which the checkpoints are merged. A 0.8 alpha would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2 @@ -81,12 +83,12 @@ def merge(self, pretrained_model_name_or_path_list: List[Union[str, os.PathLike] """ # Default kwargs from DiffusionPipeline - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) force_download = kwargs.pop("force_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) torch_dtype = kwargs.pop("torch_dtype", None) device_map = kwargs.pop("device_map", None) @@ -123,7 +125,7 @@ def merge(self, pretrained_model_name_or_path_list: List[Union[str, os.PathLike] force_download=force_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, ) config_dicts.append(config_dict) @@ -159,7 +161,7 @@ def merge(self, pretrained_model_name_or_path_list: List[Union[str, os.PathLike] resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, allow_patterns=allow_patterns, user_agent=user_agent, diff --git a/examples/community/stable_diffusion_tensorrt_img2img.py b/examples/community/stable_diffusion_tensorrt_img2img.py index 507177791f5e..e6e5e9db71d0 100755 --- a/examples/community/stable_diffusion_tensorrt_img2img.py +++ b/examples/community/stable_diffusion_tensorrt_img2img.py @@ -28,6 +28,7 @@ import tensorrt as trt import torch from huggingface_hub import snapshot_download +from huggingface_hub.utils import validate_hf_hub_args from onnx import shape_inference from polygraphy import cuda from polygraphy.backend.common import bytes_from_path @@ -50,7 +51,7 @@ StableDiffusionSafetyChecker, ) from diffusers.schedulers import DDIMScheduler -from diffusers.utils import DIFFUSERS_CACHE, logging +from diffusers.utils import logging """ @@ -778,12 +779,13 @@ def __loadModels(self): self.models["vae_encoder"] = make_VAEEncoder(self.vae, **models_args) @classmethod + @validate_hf_hub_args def set_cached_folder(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) cls.cached_folder = ( @@ -795,7 +797,7 @@ def set_cached_folder(cls, pretrained_model_name_or_path: Optional[Union[str, os resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, ) ) diff --git a/examples/community/stable_diffusion_tensorrt_inpaint.py b/examples/community/stable_diffusion_tensorrt_inpaint.py index b4e16c76159c..82b67c87fcdc 100755 --- a/examples/community/stable_diffusion_tensorrt_inpaint.py +++ b/examples/community/stable_diffusion_tensorrt_inpaint.py @@ -28,6 +28,7 @@ import tensorrt as trt import torch from huggingface_hub import snapshot_download +from huggingface_hub.utils import validate_hf_hub_args from onnx import shape_inference from polygraphy import cuda from polygraphy.backend.common import bytes_from_path @@ -51,7 +52,7 @@ ) from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint import prepare_mask_and_masked_image from diffusers.schedulers import DDIMScheduler -from diffusers.utils import DIFFUSERS_CACHE, logging +from diffusers.utils import logging """ @@ -779,12 +780,13 @@ def __loadModels(self): self.models["vae_encoder"] = make_VAEEncoder(self.vae, **models_args) @classmethod + @validate_hf_hub_args def set_cached_folder(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) cls.cached_folder = ( @@ -796,7 +798,7 @@ def set_cached_folder(cls, pretrained_model_name_or_path: Optional[Union[str, os resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, ) ) diff --git a/examples/community/stable_diffusion_tensorrt_txt2img.py b/examples/community/stable_diffusion_tensorrt_txt2img.py index c38261463384..079218d19d4a 100755 --- a/examples/community/stable_diffusion_tensorrt_txt2img.py +++ b/examples/community/stable_diffusion_tensorrt_txt2img.py @@ -27,6 +27,7 @@ import tensorrt as trt import torch from huggingface_hub import snapshot_download +from huggingface_hub.utils import validate_hf_hub_args from onnx import shape_inference from polygraphy import cuda from polygraphy.backend.common import bytes_from_path @@ -49,7 +50,7 @@ StableDiffusionSafetyChecker, ) from diffusers.schedulers import DDIMScheduler -from diffusers.utils import DIFFUSERS_CACHE, logging +from diffusers.utils import logging """ @@ -691,12 +692,13 @@ def __loadModels(self): self.models["vae"] = make_VAE(self.vae, **models_args) @classmethod + @validate_hf_hub_args def set_cached_folder(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) cls.cached_folder = ( @@ -708,7 +710,7 @@ def set_cached_folder(cls, pretrained_model_name_or_path: Optional[Union[str, os resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, ) ) diff --git a/examples/consistency_distillation/train_lcm_distill_lora_sd_wds.py b/examples/consistency_distillation/train_lcm_distill_lora_sd_wds.py index 00fd1910a657..2b7ff0c58e48 100644 --- a/examples/consistency_distillation/train_lcm_distill_lora_sd_wds.py +++ b/examples/consistency_distillation/train_lcm_distill_lora_sd_wds.py @@ -423,7 +423,7 @@ def import_model_class_from_model_name_or_path( pretrained_model_name_or_path: str, revision: str, subfolder: str = "text_encoder" ): text_encoder_config = PretrainedConfig.from_pretrained( - pretrained_model_name_or_path, subfolder=subfolder, revision=revision, use_auth_token=True + pretrained_model_name_or_path, subfolder=subfolder, revision=revision ) model_class = text_encoder_config.architectures[0] diff --git a/examples/consistency_distillation/train_lcm_distill_lora_sdxl_wds.py b/examples/consistency_distillation/train_lcm_distill_lora_sdxl_wds.py index f63333696861..eb83d451a6d4 100644 --- a/examples/consistency_distillation/train_lcm_distill_lora_sdxl_wds.py +++ b/examples/consistency_distillation/train_lcm_distill_lora_sdxl_wds.py @@ -392,7 +392,7 @@ def import_model_class_from_model_name_or_path( pretrained_model_name_or_path: str, revision: str, subfolder: str = "text_encoder" ): text_encoder_config = PretrainedConfig.from_pretrained( - pretrained_model_name_or_path, subfolder=subfolder, revision=revision, use_auth_token=True + pretrained_model_name_or_path, subfolder=subfolder, revision=revision ) model_class = text_encoder_config.architectures[0] diff --git a/examples/consistency_distillation/train_lcm_distill_sd_wds.py b/examples/consistency_distillation/train_lcm_distill_sd_wds.py index d1eda7776223..27c0e0fa0b08 100644 --- a/examples/consistency_distillation/train_lcm_distill_sd_wds.py +++ b/examples/consistency_distillation/train_lcm_distill_sd_wds.py @@ -400,7 +400,7 @@ def import_model_class_from_model_name_or_path( pretrained_model_name_or_path: str, revision: str, subfolder: str = "text_encoder" ): text_encoder_config = PretrainedConfig.from_pretrained( - pretrained_model_name_or_path, subfolder=subfolder, revision=revision, use_auth_token=True + pretrained_model_name_or_path, subfolder=subfolder, revision=revision ) model_class = text_encoder_config.architectures[0] diff --git a/examples/consistency_distillation/train_lcm_distill_sdxl_wds.py b/examples/consistency_distillation/train_lcm_distill_sdxl_wds.py index 884b2755942a..6f7f813997ee 100644 --- a/examples/consistency_distillation/train_lcm_distill_sdxl_wds.py +++ b/examples/consistency_distillation/train_lcm_distill_sdxl_wds.py @@ -414,7 +414,7 @@ def import_model_class_from_model_name_or_path( pretrained_model_name_or_path: str, revision: str, subfolder: str = "text_encoder" ): text_encoder_config = PretrainedConfig.from_pretrained( - pretrained_model_name_or_path, subfolder=subfolder, revision=revision, use_auth_token=True + pretrained_model_name_or_path, subfolder=subfolder, revision=revision ) model_class = text_encoder_config.architectures[0] diff --git a/examples/research_projects/controlnet/train_controlnet_webdataset.py b/examples/research_projects/controlnet/train_controlnet_webdataset.py index 3122a3952b33..83298298ecf8 100644 --- a/examples/research_projects/controlnet/train_controlnet_webdataset.py +++ b/examples/research_projects/controlnet/train_controlnet_webdataset.py @@ -420,7 +420,7 @@ def import_model_class_from_model_name_or_path( pretrained_model_name_or_path: str, revision: str, subfolder: str = "text_encoder" ): text_encoder_config = PretrainedConfig.from_pretrained( - pretrained_model_name_or_path, subfolder=subfolder, revision=revision, use_auth_token=True + pretrained_model_name_or_path, subfolder=subfolder, revision=revision ) model_class = text_encoder_config.architectures[0] @@ -975,7 +975,7 @@ def main(args): revision=args.revision, ) unet = UNet2DConditionModel.from_pretrained( - args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision, use_auth_token=True + args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision ) if args.controlnet_model_name_or_path: diff --git a/src/diffusers/commands/fp16_safetensors.py b/src/diffusers/commands/fp16_safetensors.py index 673e730bdabc..8373046f1880 100644 --- a/src/diffusers/commands/fp16_safetensors.py +++ b/src/diffusers/commands/fp16_safetensors.py @@ -19,6 +19,7 @@ import glob import json +import warnings from argparse import ArgumentParser, Namespace from importlib import import_module @@ -32,12 +33,12 @@ def conversion_command_factory(args: Namespace): - return FP16SafetensorsCommand( - args.ckpt_id, - args.fp16, - args.use_safetensors, - args.use_auth_token, - ) + if args.use_auth_token: + warnings.warn( + "The `--use_auth_token` flag is deprecated and will be removed in a future version. Authentication is now" + " handled automatically if user is logged in." + ) + return FP16SafetensorsCommand(args.ckpt_id, args.fp16, args.use_safetensors) class FP16SafetensorsCommand(BaseDiffusersCLICommand): @@ -62,7 +63,7 @@ def register_subcommand(parser: ArgumentParser): ) conversion_parser.set_defaults(func=conversion_command_factory) - def __init__(self, ckpt_id: str, fp16: bool, use_safetensors: bool, use_auth_token: bool): + def __init__(self, ckpt_id: str, fp16: bool, use_safetensors: bool): self.logger = logging.get_logger("diffusers-cli/fp16_safetensors") self.ckpt_id = ckpt_id self.local_ckpt_dir = f"/tmp/{ckpt_id}" @@ -75,8 +76,6 @@ def __init__(self, ckpt_id: str, fp16: bool, use_safetensors: bool, use_auth_tok "When `use_safetensors` and `fp16` both are False, then this command is of no use." ) - self.use_auth_token = use_auth_token - def run(self): if version.parse(huggingface_hub.__version__) < version.parse("0.9.0"): raise ImportError( @@ -87,7 +86,7 @@ def run(self): from huggingface_hub import create_commit from huggingface_hub._commit_api import CommitOperationAdd - model_index = hf_hub_download(repo_id=self.ckpt_id, filename="model_index.json", token=self.use_auth_token) + model_index = hf_hub_download(repo_id=self.ckpt_id, filename="model_index.json") with open(model_index, "r") as f: pipeline_class_name = json.load(f)["_class_name"] pipeline_class = getattr(import_module("diffusers"), pipeline_class_name) @@ -96,7 +95,7 @@ def run(self): # Load the appropriate pipeline. We could have use `DiffusionPipeline` # here, but just to avoid any rough edge cases. pipeline = pipeline_class.from_pretrained( - self.ckpt_id, torch_dtype=torch.float16 if self.fp16 else torch.float32, use_auth_token=self.use_auth_token + self.ckpt_id, torch_dtype=torch.float16 if self.fp16 else torch.float32 ) pipeline.save_pretrained( self.local_ckpt_dir, diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index 1b91bfda3058..38cd77e6ef13 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -27,12 +27,16 @@ import numpy as np from huggingface_hub import create_repo, hf_hub_download -from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, + validate_hf_hub_args, +) from requests import HTTPError from . import __version__ from .utils import ( - DIFFUSERS_CACHE, HUGGINGFACE_CO_RESOLVE_ENDPOINT, DummyObject, deprecate, @@ -275,6 +279,7 @@ def get_config_dict(cls, *args, **kwargs): return cls.load_config(*args, **kwargs) @classmethod + @validate_hf_hub_args def load_config( cls, pretrained_model_name_or_path: Union[str, os.PathLike], @@ -311,7 +316,7 @@ def load_config( local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -329,11 +334,11 @@ def load_config( A dictionary of all the parameters stored in a JSON configuration file. """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) _ = kwargs.pop("mirror", None) @@ -376,7 +381,7 @@ def load_config( proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, user_agent=user_agent, subfolder=subfolder, revision=revision, @@ -385,8 +390,7 @@ def load_config( raise EnvironmentError( f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier" " listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a" - " token having permission to this repo with `use_auth_token` or log in with `huggingface-cli" - " login`." + " token having permission to this repo with `token` or log in with `huggingface-cli login`." ) except RevisionNotFoundError: raise EnvironmentError( diff --git a/src/diffusers/loaders/ip_adapter.py b/src/diffusers/loaders/ip_adapter.py index 32c558554be2..158bde436374 100644 --- a/src/diffusers/loaders/ip_adapter.py +++ b/src/diffusers/loaders/ip_adapter.py @@ -15,11 +15,10 @@ from typing import Dict, Union import torch +from huggingface_hub.utils import validate_hf_hub_args from safetensors import safe_open from ..utils import ( - DIFFUSERS_CACHE, - HF_HUB_OFFLINE, _get_model_file, is_transformers_available, logging, @@ -43,6 +42,7 @@ class IPAdapterMixin: """Mixin for handling IP Adapters.""" + @validate_hf_hub_args def load_ip_adapter( self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], @@ -77,7 +77,7 @@ def load_ip_adapter( local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -88,12 +88,12 @@ def load_ip_adapter( """ # Load the main state dict first. - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) user_agent = { @@ -110,7 +110,7 @@ def load_ip_adapter( resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, diff --git a/src/diffusers/loaders/lora.py b/src/diffusers/loaders/lora.py index dde717959f8e..3955fc2a1395 100644 --- a/src/diffusers/loaders/lora.py +++ b/src/diffusers/loaders/lora.py @@ -18,14 +18,13 @@ import safetensors import torch from huggingface_hub import model_info +from huggingface_hub.utils import validate_hf_hub_args from packaging import version from torch import nn from .. import __version__ from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT, load_model_dict_into_meta from ..utils import ( - DIFFUSERS_CACHE, - HF_HUB_OFFLINE, USE_PEFT_BACKEND, _get_model_file, convert_state_dict_to_diffusers, @@ -132,6 +131,7 @@ def load_lora_weights( ) @classmethod + @validate_hf_hub_args def lora_state_dict( cls, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], @@ -174,7 +174,7 @@ def lora_state_dict( local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -195,12 +195,12 @@ def lora_state_dict( """ # Load the main state dict first which has the LoRA layers for either of # UNet and text encoder or both. - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", None) weight_name = kwargs.pop("weight_name", None) @@ -239,7 +239,7 @@ def lora_state_dict( resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, @@ -265,7 +265,7 @@ def lora_state_dict( resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py index bf100e7f2c81..a49280adfcfe 100644 --- a/src/diffusers/loaders/single_file.py +++ b/src/diffusers/loaders/single_file.py @@ -18,10 +18,9 @@ import requests import torch from huggingface_hub import hf_hub_download +from huggingface_hub.utils import validate_hf_hub_args from ..utils import ( - DIFFUSERS_CACHE, - HF_HUB_OFFLINE, deprecate, is_accelerate_available, is_omegaconf_available, @@ -52,6 +51,7 @@ def from_ckpt(cls, *args, **kwargs): return cls.from_single_file(*args, **kwargs) @classmethod + @validate_hf_hub_args def from_single_file(cls, pretrained_model_link_or_path, **kwargs): r""" Instantiate a [`DiffusionPipeline`] from pretrained pipeline weights saved in the `.ckpt` or `.safetensors` @@ -81,7 +81,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -154,12 +154,12 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): original_config_file = kwargs.pop("original_config_file", None) config_files = kwargs.pop("config_files", None) - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) force_download = kwargs.pop("force_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) extract_ema = kwargs.pop("extract_ema", False) image_size = kwargs.pop("image_size", None) @@ -253,7 +253,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, force_download=force_download, ) @@ -293,6 +293,7 @@ class FromOriginalVAEMixin: """ @classmethod + @validate_hf_hub_args def from_single_file(cls, pretrained_model_link_or_path, **kwargs): r""" Instantiate a [`AutoencoderKL`] from pretrained ControlNet weights saved in the original `.ckpt` or @@ -322,7 +323,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to True, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -379,12 +380,12 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): ) config_file = kwargs.pop("config_file", None) - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) force_download = kwargs.pop("force_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) image_size = kwargs.pop("image_size", None) scaling_factor = kwargs.pop("scaling_factor", None) @@ -425,7 +426,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, force_download=force_download, ) @@ -490,6 +491,7 @@ class FromOriginalControlnetMixin: """ @classmethod + @validate_hf_hub_args def from_single_file(cls, pretrained_model_link_or_path, **kwargs): r""" Instantiate a [`ControlNetModel`] from pretrained ControlNet weights saved in the original `.ckpt` or @@ -519,7 +521,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to True, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -555,12 +557,12 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): from ..pipelines.stable_diffusion.convert_from_ckpt import download_controlnet_from_original_ckpt config_file = kwargs.pop("config_file", None) - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) force_download = kwargs.pop("force_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) num_in_channels = kwargs.pop("num_in_channels", None) use_linear_projection = kwargs.pop("use_linear_projection", None) revision = kwargs.pop("revision", None) @@ -603,7 +605,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, force_download=force_download, ) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index d03bd74d5250..96aa1bce7cbe 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -15,16 +15,10 @@ import safetensors import torch +from huggingface_hub.utils import validate_hf_hub_args from torch import nn -from ..utils import ( - DIFFUSERS_CACHE, - HF_HUB_OFFLINE, - _get_model_file, - is_accelerate_available, - is_transformers_available, - logging, -) +from ..utils import _get_model_file, is_accelerate_available, is_transformers_available, logging if is_transformers_available(): @@ -39,13 +33,14 @@ TEXT_INVERSION_NAME_SAFE = "learned_embeds.safetensors" +@validate_hf_hub_args def load_textual_inversion_state_dicts(pretrained_model_name_or_paths, **kwargs): - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", None) weight_name = kwargs.pop("weight_name", None) @@ -79,7 +74,7 @@ def load_textual_inversion_state_dicts(pretrained_model_name_or_paths, **kwargs) resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, @@ -100,7 +95,7 @@ def load_textual_inversion_state_dicts(pretrained_model_name_or_paths, **kwargs) resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, @@ -267,6 +262,7 @@ def _extend_tokens_and_embeddings(tokens, embeddings, tokenizer): return all_tokens, all_embeddings + @validate_hf_hub_args def load_textual_inversion( self, pretrained_model_name_or_path: Union[str, List[str], Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]], @@ -320,7 +316,7 @@ def load_textual_inversion( local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): diff --git a/src/diffusers/loaders/unet.py b/src/diffusers/loaders/unet.py index 9d559a4b4af8..b155f595e740 100644 --- a/src/diffusers/loaders/unet.py +++ b/src/diffusers/loaders/unet.py @@ -19,13 +19,12 @@ import safetensors import torch import torch.nn.functional as F +from huggingface_hub.utils import validate_hf_hub_args from torch import nn from ..models.embeddings import ImageProjection, Resampler from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT, load_model_dict_into_meta from ..utils import ( - DIFFUSERS_CACHE, - HF_HUB_OFFLINE, USE_PEFT_BACKEND, _get_model_file, delete_adapter_layers, @@ -62,6 +61,7 @@ class UNet2DConditionLoadersMixin: text_encoder_name = TEXT_ENCODER_NAME unet_name = UNET_NAME + @validate_hf_hub_args def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): r""" Load pretrained attention processor layers into [`UNet2DConditionModel`]. Attention processor layers have to be @@ -95,7 +95,7 @@ def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): @@ -130,12 +130,12 @@ def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict from ..models.attention_processor import CustomDiffusionAttnProcessor from ..models.lora import LoRACompatibleConv, LoRACompatibleLinear, LoRAConv2dLayer, LoRALinearLayer - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", None) weight_name = kwargs.pop("weight_name", None) @@ -184,7 +184,7 @@ def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, @@ -204,7 +204,7 @@ def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, diff --git a/src/diffusers/models/__init__.py b/src/diffusers/models/__init__.py index 49ee3ee6af6b..e3794939e25e 100644 --- a/src/diffusers/models/__init__.py +++ b/src/diffusers/models/__init__.py @@ -33,8 +33,8 @@ _import_structure["consistency_decoder_vae"] = ["ConsistencyDecoderVAE"] _import_structure["controlnet"] = ["ControlNetModel"] _import_structure["dual_transformer_2d"] = ["DualTransformer2DModel"] - _import_structure["modeling_utils"] = ["ModelMixin"] _import_structure["embeddings"] = ["ImageProjection"] + _import_structure["modeling_utils"] = ["ModelMixin"] _import_structure["prior_transformer"] = ["PriorTransformer"] _import_structure["t5_film_transformer"] = ["T5FilmDecoder"] _import_structure["transformer_2d"] = ["Transformer2DModel"] diff --git a/src/diffusers/models/modeling_flax_utils.py b/src/diffusers/models/modeling_flax_utils.py index 0ea0819ca07a..1770cae494ed 100644 --- a/src/diffusers/models/modeling_flax_utils.py +++ b/src/diffusers/models/modeling_flax_utils.py @@ -24,13 +24,17 @@ from flax.serialization import from_bytes, to_bytes from flax.traverse_util import flatten_dict, unflatten_dict from huggingface_hub import create_repo, hf_hub_download -from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, + validate_hf_hub_args, +) from requests import HTTPError from .. import __version__, is_torch_available from ..utils import ( CONFIG_NAME, - DIFFUSERS_CACHE, FLAX_WEIGHTS_NAME, HUGGINGFACE_CO_RESOLVE_ENDPOINT, WEIGHTS_NAME, @@ -197,6 +201,7 @@ def init_weights(self, rng: jax.Array) -> Dict: raise NotImplementedError(f"init_weights method has to be implemented for {self}") @classmethod + @validate_hf_hub_args def from_pretrained( cls, pretrained_model_name_or_path: Union[str, os.PathLike], @@ -288,13 +293,13 @@ def from_pretrained( ``` """ config = kwargs.pop("config", None) - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) from_pt = kwargs.pop("from_pt", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", None) @@ -314,7 +319,7 @@ def from_pretrained( resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, **kwargs, @@ -359,7 +364,7 @@ def from_pretrained( proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, user_agent=user_agent, subfolder=subfolder, revision=revision, @@ -369,7 +374,7 @@ def from_pretrained( raise EnvironmentError( f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " - "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli " + "token having permission to this repo with `token` or log in with `huggingface-cli " "login`." ) except RevisionNotFoundError: diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 644c52f103fa..546c5b20f937 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -25,14 +25,13 @@ import safetensors import torch from huggingface_hub import create_repo +from huggingface_hub.utils import validate_hf_hub_args from torch import Tensor, nn from .. import __version__ from ..utils import ( CONFIG_NAME, - DIFFUSERS_CACHE, FLAX_WEIGHTS_NAME, - HF_HUB_OFFLINE, MIN_PEFT_VERSION, SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, @@ -535,6 +534,7 @@ def save_pretrained( ) @classmethod + @validate_hf_hub_args def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): r""" Instantiate a pretrained PyTorch model from a pretrained model configuration. @@ -571,7 +571,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P local_files_only(`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -640,15 +640,15 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. ``` """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) force_download = kwargs.pop("force_download", False) from_flax = kwargs.pop("from_flax", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) output_loading_info = kwargs.pop("output_loading_info", False) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) torch_dtype = kwargs.pop("torch_dtype", None) subfolder = kwargs.pop("subfolder", None) @@ -718,7 +718,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, device_map=device_map, @@ -740,7 +740,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, @@ -763,7 +763,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, @@ -782,7 +782,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, user_agent=user_agent, diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index a7c6cd82c8e7..00738be3f374 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -16,8 +16,9 @@ import inspect from collections import OrderedDict +from huggingface_hub.utils import validate_hf_hub_args + from ..configuration_utils import ConfigMixin -from ..utils import DIFFUSERS_CACHE from .controlnet import ( StableDiffusionControlNetImg2ImgPipeline, StableDiffusionControlNetInpaintPipeline, @@ -195,6 +196,7 @@ def __init__(self, *args, **kwargs): ) @classmethod + @validate_hf_hub_args def from_pretrained(cls, pretrained_model_or_path, **kwargs): r""" Instantiates a text-to-image Pytorch diffusion pipeline from pretrained pipeline weight. @@ -246,7 +248,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -310,11 +312,11 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> image = pipeline(prompt).images[0] ``` """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) @@ -323,7 +325,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): "force_download": force_download, "resume_download": resume_download, "proxies": proxies, - "use_auth_token": use_auth_token, + "token": token, "local_files_only": local_files_only, "revision": revision, } @@ -466,6 +468,7 @@ def __init__(self, *args, **kwargs): ) @classmethod + @validate_hf_hub_args def from_pretrained(cls, pretrained_model_or_path, **kwargs): r""" Instantiates a image-to-image Pytorch diffusion pipeline from pretrained pipeline weight. @@ -518,7 +521,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -582,11 +585,11 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> image = pipeline(prompt, image).images[0] ``` """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) @@ -595,7 +598,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): "force_download": force_download, "resume_download": resume_download, "proxies": proxies, - "use_auth_token": use_auth_token, + "token": token, "local_files_only": local_files_only, "revision": revision, } @@ -742,6 +745,7 @@ def __init__(self, *args, **kwargs): ) @classmethod + @validate_hf_hub_args def from_pretrained(cls, pretrained_model_or_path, **kwargs): r""" Instantiates a inpainting Pytorch diffusion pipeline from pretrained pipeline weight. @@ -793,7 +797,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -857,11 +861,11 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> image = pipeline(prompt, image=init_image, mask_image=mask_image).images[0] ``` """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) @@ -870,7 +874,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): "force_download": force_download, "resume_download": resume_download, "proxies": proxies, - "use_auth_token": use_auth_token, + "token": token, "local_files_only": local_files_only, "revision": revision, } diff --git a/src/diffusers/pipelines/onnx_utils.py b/src/diffusers/pipelines/onnx_utils.py index 07c32e4e84bf..43827c7a61f2 100644 --- a/src/diffusers/pipelines/onnx_utils.py +++ b/src/diffusers/pipelines/onnx_utils.py @@ -22,6 +22,7 @@ import numpy as np from huggingface_hub import hf_hub_download +from huggingface_hub.utils import validate_hf_hub_args from ..utils import ONNX_EXTERNAL_WEIGHTS_NAME, ONNX_WEIGHTS_NAME, is_onnx_available, logging @@ -130,10 +131,11 @@ def save_pretrained( self._save_pretrained(save_directory, **kwargs) @classmethod + @validate_hf_hub_args def _from_pretrained( cls, model_id: Union[str, Path], - use_auth_token: Optional[Union[bool, str, None]] = None, + token: Optional[Union[bool, str, None]] = None, revision: Optional[Union[str, None]] = None, force_download: bool = False, cache_dir: Optional[str] = None, @@ -148,7 +150,7 @@ def _from_pretrained( Arguments: model_id (`str` or `Path`): Directory from which to load - use_auth_token (`str` or `bool`): + token (`str` or `bool`): Is needed to load models from a private or gated repository revision (`str`): Revision is the specific model version to use. It can be a branch name, a tag name, or a commit id @@ -179,7 +181,7 @@ def _from_pretrained( model_cache_path = hf_hub_download( repo_id=model_id, filename=model_file_name, - use_auth_token=use_auth_token, + token=token, revision=revision, cache_dir=cache_dir, force_download=force_download, @@ -190,11 +192,12 @@ def _from_pretrained( return cls(model=model, **kwargs) @classmethod + @validate_hf_hub_args def from_pretrained( cls, model_id: Union[str, Path], force_download: bool = True, - use_auth_token: Optional[str] = None, + token: Optional[str] = None, cache_dir: Optional[str] = None, **model_kwargs, ): @@ -207,6 +210,6 @@ def from_pretrained( revision=revision, cache_dir=cache_dir, force_download=force_download, - use_auth_token=use_auth_token, + token=token, **model_kwargs, ) diff --git a/src/diffusers/pipelines/pipeline_flax_utils.py b/src/diffusers/pipelines/pipeline_flax_utils.py index 2e25a40295b4..7ddde8f1ca5d 100644 --- a/src/diffusers/pipelines/pipeline_flax_utils.py +++ b/src/diffusers/pipelines/pipeline_flax_utils.py @@ -24,6 +24,7 @@ import PIL.Image from flax.core.frozen_dict import FrozenDict from huggingface_hub import create_repo, snapshot_download +from huggingface_hub.utils import validate_hf_hub_args from PIL import Image from tqdm.auto import tqdm @@ -32,7 +33,6 @@ from ..schedulers.scheduling_utils_flax import SCHEDULER_CONFIG_NAME, FlaxSchedulerMixin from ..utils import ( CONFIG_NAME, - DIFFUSERS_CACHE, BaseOutput, PushToHubMixin, http_user_agent, @@ -227,6 +227,7 @@ class implements both a save and loading method. The pipeline is easily reloaded ) @classmethod + @validate_hf_hub_args def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): r""" Instantiate a Flax-based diffusion pipeline from pretrained pipeline weights. @@ -264,7 +265,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -314,11 +315,11 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P >>> dpm_params["scheduler"] = dpmpp_state ``` """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) local_files_only = kwargs.pop("local_files_only", False) - use_auth_token = kwargs.pop("use_auth_token", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) from_pt = kwargs.pop("from_pt", False) use_memory_efficient_attention = kwargs.pop("use_memory_efficient_attention", False) @@ -334,7 +335,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, ) # make sure we only download sub-folders and `diffusers` filenames @@ -365,7 +366,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns, diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py index 7d889f3afa4c..e7a795365ad3 100644 --- a/src/diffusers/pipelines/pipeline_utils.py +++ b/src/diffusers/pipelines/pipeline_utils.py @@ -28,7 +28,14 @@ import numpy as np import PIL.Image import torch -from huggingface_hub import ModelCard, create_repo, hf_hub_download, model_info, snapshot_download +from huggingface_hub import ( + ModelCard, + create_repo, + hf_hub_download, + model_info, + snapshot_download, +) +from huggingface_hub.utils import validate_hf_hub_args from packaging import version from requests.exceptions import HTTPError from tqdm.auto import tqdm @@ -40,8 +47,6 @@ from ..utils import ( CONFIG_NAME, DEPRECATED_REVISION_ARGS, - DIFFUSERS_CACHE, - HF_HUB_OFFLINE, SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, BaseOutput, @@ -249,10 +254,11 @@ def convert_to_variant(filename): return usable_filenames, variant_filenames -def warn_deprecated_model_variant(pretrained_model_name_or_path, use_auth_token, variant, revision, model_filenames): +@validate_hf_hub_args +def warn_deprecated_model_variant(pretrained_model_name_or_path, token, variant, revision, model_filenames): info = model_info( pretrained_model_name_or_path, - use_auth_token=use_auth_token, + token=token, revision=None, ) filenames = {sibling.rfilename for sibling in info.siblings} @@ -375,7 +381,6 @@ def _get_pipeline_class( custom_pipeline, module_file=file_name, class_name=class_name, - repo_id=repo_id, cache_dir=cache_dir, revision=revision, ) @@ -909,6 +914,7 @@ def dtype(self) -> torch.dtype: return torch.float32 @classmethod + @validate_hf_hub_args def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): r""" Instantiate a PyTorch diffusion pipeline from pretrained pipeline weights. @@ -976,7 +982,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -1056,12 +1062,12 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P >>> pipeline.scheduler = scheduler ``` """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) force_download = kwargs.pop("force_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) from_flax = kwargs.pop("from_flax", False) torch_dtype = kwargs.pop("torch_dtype", None) @@ -1094,7 +1100,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P force_download=force_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, from_flax=from_flax, use_safetensors=use_safetensors, @@ -1299,7 +1305,7 @@ def load_module(name, value): "force_download": force_download, "proxies": proxies, "local_files_only": local_files_only, - "use_auth_token": use_auth_token, + "token": token, "revision": revision, "torch_dtype": torch_dtype, "custom_pipeline": custom_pipeline, @@ -1529,6 +1535,7 @@ def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Un cpu_offload(model, device, offload_buffers=offload_buffers) @classmethod + @validate_hf_hub_args def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: r""" Download and cache a PyTorch diffusion pipeline from pretrained pipeline weights. @@ -1576,7 +1583,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): @@ -1619,12 +1626,12 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: """ - cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + cache_dir = kwargs.pop("cache_dir", None) resume_download = kwargs.pop("resume_download", False) force_download = kwargs.pop("force_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) - use_auth_token = kwargs.pop("use_auth_token", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) revision = kwargs.pop("revision", None) from_flax = kwargs.pop("from_flax", False) custom_pipeline = kwargs.pop("custom_pipeline", None) @@ -1646,11 +1653,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: model_info_call_error: Optional[Exception] = None if not local_files_only: try: - info = model_info( - pretrained_model_name, - use_auth_token=use_auth_token, - revision=revision, - ) + info = model_info(pretrained_model_name, token=token, revision=revision) except HTTPError as e: logger.warn(f"Couldn't connect to the Hub: {e}.\nWill try to load from local cache.") local_files_only = True @@ -1665,7 +1668,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: proxies=proxies, force_download=force_download, resume_download=resume_download, - use_auth_token=use_auth_token, + token=token, ) config_dict = cls._dict_from_json_file(config_file) @@ -1715,9 +1718,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: if revision in DEPRECATED_REVISION_ARGS and version.parse( version.parse(__version__).base_version ) >= version.parse("0.22.0"): - warn_deprecated_model_variant( - pretrained_model_name, use_auth_token, variant, revision, model_filenames - ) + warn_deprecated_model_variant(pretrained_model_name, token, variant, revision, model_filenames) model_folder_names = {os.path.split(f)[0] for f in model_filenames if os.path.split(f)[0] in folder_names} @@ -1859,7 +1860,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns, @@ -1883,7 +1884,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: "force_download": force_download, "proxies": proxies, "local_files_only": local_files_only, - "use_auth_token": use_auth_token, + "token": token, "variant": variant, "use_safetensors": use_safetensors, } diff --git a/src/diffusers/schedulers/scheduling_utils.py b/src/diffusers/schedulers/scheduling_utils.py index 9d9472a9063f..9eadadb1d26f 100644 --- a/src/diffusers/schedulers/scheduling_utils.py +++ b/src/diffusers/schedulers/scheduling_utils.py @@ -18,6 +18,7 @@ from typing import Optional, Union import torch +from huggingface_hub.utils import validate_hf_hub_args from ..utils import BaseOutput, PushToHubMixin @@ -81,6 +82,7 @@ class SchedulerMixin(PushToHubMixin): has_compatibles = True @classmethod + @validate_hf_hub_args def from_pretrained( cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, @@ -120,7 +122,7 @@ def from_pretrained( local_files_only(`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): diff --git a/src/diffusers/schedulers/scheduling_utils_flax.py b/src/diffusers/schedulers/scheduling_utils_flax.py index ccec121d3094..5e7524cf293d 100644 --- a/src/diffusers/schedulers/scheduling_utils_flax.py +++ b/src/diffusers/schedulers/scheduling_utils_flax.py @@ -20,6 +20,7 @@ import flax import jax.numpy as jnp +from huggingface_hub.utils import validate_hf_hub_args from ..utils import BaseOutput, PushToHubMixin @@ -70,6 +71,7 @@ class FlaxSchedulerMixin(PushToHubMixin): has_compatibles = True @classmethod + @validate_hf_hub_args def from_pretrained( cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, @@ -110,7 +112,7 @@ def from_pretrained( Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (i.e., do not try to download the model). - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `transformers-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): diff --git a/src/diffusers/utils/__init__.py b/src/diffusers/utils/__init__.py index c1385d584724..181905bf5ac0 100644 --- a/src/diffusers/utils/__init__.py +++ b/src/diffusers/utils/__init__.py @@ -21,7 +21,6 @@ from .constants import ( CONFIG_NAME, DEPRECATED_REVISION_ARGS, - DIFFUSERS_CACHE, DIFFUSERS_DYNAMIC_MODULE_NAME, FLAX_WEIGHTS_NAME, HF_MODULES_CACHE, @@ -38,7 +37,6 @@ from .dynamic_modules_utils import get_class_from_dynamic_module from .export_utils import export_to_gif, export_to_obj, export_to_ply, export_to_video from .hub_utils import ( - HF_HUB_OFFLINE, PushToHubMixin, _add_variant, _get_model_file, diff --git a/src/diffusers/utils/constants.py b/src/diffusers/utils/constants.py index 608a751fb8d6..8850da073e95 100644 --- a/src/diffusers/utils/constants.py +++ b/src/diffusers/utils/constants.py @@ -14,15 +14,13 @@ import importlib import os -from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home +from huggingface_hub.constants import HF_HOME from packaging import version from ..dependency_versions_check import dep_version_check from .import_utils import ENV_VARS_TRUE_VALUES, is_peft_available, is_transformers_available -default_cache_path = HUGGINGFACE_HUB_CACHE - MIN_PEFT_VERSION = "0.6.0" MIN_TRANSFORMERS_VERSION = "4.34.0" _CHECK_PEFT = os.environ.get("_CHECK_PEFT", "1") in ENV_VARS_TRUE_VALUES @@ -35,9 +33,8 @@ SAFETENSORS_WEIGHTS_NAME = "diffusion_pytorch_model.safetensors" ONNX_EXTERNAL_WEIGHTS_NAME = "weights.pb" HUGGINGFACE_CO_RESOLVE_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") -DIFFUSERS_CACHE = default_cache_path DIFFUSERS_DYNAMIC_MODULE_NAME = "diffusers_modules" -HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules")) +HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(HF_HOME, "modules")) DEPRECATED_REVISION_ARGS = ["fp16", "non-ema"] # Below should be `True` if the current version of `peft` and `transformers` are compatible with diff --git a/src/diffusers/utils/dynamic_modules_utils.py b/src/diffusers/utils/dynamic_modules_utils.py index d668cb40c631..f13dd4799be3 100644 --- a/src/diffusers/utils/dynamic_modules_utils.py +++ b/src/diffusers/utils/dynamic_modules_utils.py @@ -25,7 +25,8 @@ from typing import Dict, Optional, Union from urllib import request -from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info +from huggingface_hub import cached_download, hf_hub_download, model_info +from huggingface_hub.utils import validate_hf_hub_args from packaging import version from .. import __version__ @@ -194,6 +195,7 @@ def find_pipeline_class(loaded_module): return pipeline_class +@validate_hf_hub_args def get_cached_module_file( pretrained_model_name_or_path: Union[str, os.PathLike], module_file: str, @@ -201,7 +203,7 @@ def get_cached_module_file( force_download: bool = False, resume_download: bool = False, proxies: Optional[Dict[str, str]] = None, - use_auth_token: Optional[Union[bool, str]] = None, + token: Optional[Union[bool, str]] = None, revision: Optional[str] = None, local_files_only: bool = False, ): @@ -232,7 +234,7 @@ def get_cached_module_file( proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `transformers-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -244,7 +246,7 @@ def get_cached_module_file( - You may pass a token in `use_auth_token` if you are not logged in (`huggingface-cli long`) and want to use private + You may pass a token in `token` if you are not logged in (`huggingface-cli login`) and want to use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models). @@ -289,7 +291,7 @@ def get_cached_module_file( proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=False, + token=False, ) submodule = "git" module_file = pretrained_model_name_or_path + ".py" @@ -307,7 +309,7 @@ def get_cached_module_file( proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, ) submodule = os.path.join("local", "--".join(pretrained_model_name_or_path.split("/"))) except EnvironmentError: @@ -332,13 +334,6 @@ def get_cached_module_file( else: # Get the commit hash # TODO: we will get this info in the etag soon, so retrieve it from there and not here. - if isinstance(use_auth_token, str): - token = use_auth_token - elif use_auth_token is True: - token = HfFolder.get_token() - else: - token = None - commit_hash = model_info(pretrained_model_name_or_path, revision=revision, token=token).sha # The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the @@ -359,13 +354,14 @@ def get_cached_module_file( force_download=force_download, resume_download=resume_download, proxies=proxies, - use_auth_token=use_auth_token, + token=token, revision=revision, local_files_only=local_files_only, ) return os.path.join(full_submodule, module_file) +@validate_hf_hub_args def get_class_from_dynamic_module( pretrained_model_name_or_path: Union[str, os.PathLike], module_file: str, @@ -374,7 +370,7 @@ def get_class_from_dynamic_module( force_download: bool = False, resume_download: bool = False, proxies: Optional[Dict[str, str]] = None, - use_auth_token: Optional[Union[bool, str]] = None, + token: Optional[Union[bool, str]] = None, revision: Optional[str] = None, local_files_only: bool = False, **kwargs, @@ -414,7 +410,7 @@ def get_class_from_dynamic_module( proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `transformers-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -426,7 +422,7 @@ def get_class_from_dynamic_module( - You may pass a token in `use_auth_token` if you are not logged in (`huggingface-cli long`) and want to use private + You may pass a token in `token` if you are not logged in (`huggingface-cli login`) and want to use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models). @@ -449,7 +445,7 @@ def get_class_from_dynamic_module( force_download=force_download, resume_download=resume_download, proxies=proxies, - use_auth_token=use_auth_token, + token=token, revision=revision, local_files_only=local_files_only, ) diff --git a/src/diffusers/utils/hub_utils.py b/src/diffusers/utils/hub_utils.py index 5cd041fbc39f..d762f015a7bc 100644 --- a/src/diffusers/utils/hub_utils.py +++ b/src/diffusers/utils/hub_utils.py @@ -25,20 +25,21 @@ from uuid import uuid4 from huggingface_hub import ( - HfFolder, ModelCard, ModelCardData, create_repo, + get_full_repo_name, hf_hub_download, upload_folder, - whoami, ) +from huggingface_hub.constants import HF_HUB_CACHE, HF_HUB_DISABLE_TELEMETRY, HF_HUB_OFFLINE from huggingface_hub.file_download import REGEX_COMMIT_HASH from huggingface_hub.utils import ( EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError, is_jinja_available, + validate_hf_hub_args, ) from packaging import version from requests import HTTPError @@ -46,7 +47,6 @@ from .. import __version__ from .constants import ( DEPRECATED_REVISION_ARGS, - DIFFUSERS_CACHE, HUGGINGFACE_CO_RESOLVE_ENDPOINT, SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, @@ -69,9 +69,6 @@ MODEL_CARD_TEMPLATE_PATH = Path(__file__).parent / "model_card_template.md" SESSION_ID = uuid4().hex -HF_HUB_OFFLINE = os.getenv("HF_HUB_OFFLINE", "").upper() in ENV_VARS_TRUE_VALUES -DISABLE_TELEMETRY = os.getenv("DISABLE_TELEMETRY", "").upper() in ENV_VARS_TRUE_VALUES -HUGGINGFACE_CO_TELEMETRY = HUGGINGFACE_CO_RESOLVE_ENDPOINT + "/api/telemetry/" def http_user_agent(user_agent: Union[Dict, str, None] = None) -> str: @@ -79,7 +76,7 @@ def http_user_agent(user_agent: Union[Dict, str, None] = None) -> str: Formats a user-agent string with basic info about a request. """ ua = f"diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}" - if DISABLE_TELEMETRY or HF_HUB_OFFLINE: + if HF_HUB_DISABLE_TELEMETRY or HF_HUB_OFFLINE: return ua + "; telemetry/off" if is_torch_available(): ua += f"; torch/{_torch_version}" @@ -98,16 +95,6 @@ def http_user_agent(user_agent: Union[Dict, str, None] = None) -> str: return ua -def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None): - if token is None: - token = HfFolder.get_token() - if organization is None: - username = whoami(token)["name"] - return f"{username}/{model_id}" - else: - return f"{organization}/{model_id}" - - def create_model_card(args, model_name): if not is_jinja_available(): raise ValueError( @@ -183,7 +170,7 @@ def extract_commit_hash(resolved_file: Optional[str], commit_hash: Optional[str] def move_cache(old_cache_dir: Optional[str] = None, new_cache_dir: Optional[str] = None) -> None: if new_cache_dir is None: - new_cache_dir = DIFFUSERS_CACHE + new_cache_dir = HF_HUB_CACHE if old_cache_dir is None: old_cache_dir = old_diffusers_cache @@ -203,7 +190,7 @@ def move_cache(old_cache_dir: Optional[str] = None, new_cache_dir: Optional[str] # At this point, old_cache_dir contains symlinks to the new cache (it can still be used). -cache_version_file = os.path.join(DIFFUSERS_CACHE, "version_diffusers_cache.txt") +cache_version_file = os.path.join(HF_HUB_CACHE, "version_diffusers_cache.txt") if not os.path.isfile(cache_version_file): cache_version = 0 else: @@ -233,12 +220,12 @@ def move_cache(old_cache_dir: Optional[str] = None, new_cache_dir: Optional[str] if cache_version < 1: try: - os.makedirs(DIFFUSERS_CACHE, exist_ok=True) + os.makedirs(HF_HUB_CACHE, exist_ok=True) with open(cache_version_file, "w") as f: f.write("1") except Exception: logger.warning( - f"There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure " + f"There was a problem when trying to write in your cache folder ({HF_HUB_CACHE}). Please, ensure " "the directory exists and can be written to." ) @@ -252,20 +239,21 @@ def _add_variant(weights_name: str, variant: Optional[str] = None) -> str: return weights_name +@validate_hf_hub_args def _get_model_file( - pretrained_model_name_or_path, + pretrained_model_name_or_path: Union[str, Path], *, - weights_name, - subfolder, - cache_dir, - force_download, - proxies, - resume_download, - local_files_only, - use_auth_token, - user_agent, - revision, - commit_hash=None, + weights_name: str, + subfolder: Optional[str], + cache_dir: Optional[str], + force_download: bool, + proxies: Optional[Dict], + resume_download: bool, + local_files_only: bool, + token: Optional[str], + user_agent: Union[Dict, str, None], + revision: Optional[str], + commit_hash: Optional[str] = None, ): pretrained_model_name_or_path = str(pretrained_model_name_or_path) if os.path.isfile(pretrained_model_name_or_path): @@ -300,7 +288,7 @@ def _get_model_file( proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, user_agent=user_agent, subfolder=subfolder, revision=revision or commit_hash, @@ -325,7 +313,7 @@ def _get_model_file( proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, user_agent=user_agent, subfolder=subfolder, revision=revision or commit_hash, @@ -336,7 +324,7 @@ def _get_model_file( raise EnvironmentError( f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " - "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli " + "token having permission to this repo with `token` or log in with `huggingface-cli " "login`." ) except RevisionNotFoundError: