From 91042afe6c3a4e0d236dd30b92e123a0cb00d3fe Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Mon, 2 Sep 2024 20:50:39 +0530 Subject: [PATCH 01/12] [Feature] Update Repo Settings --- docs/source/en/guides/repository.md | 14 +++ src/huggingface_hub/__init__.py | 4 +- src/huggingface_hub/hf_api.py | 157 +++++++--------------------- tests/test_hf_api.py | 22 ++++ 4 files changed, 74 insertions(+), 123 deletions(-) diff --git a/docs/source/en/guides/repository.md b/docs/source/en/guides/repository.md index 4e6aab5b8d..733255868e 100644 --- a/docs/source/en/guides/repository.md +++ b/docs/source/en/guides/repository.md @@ -155,6 +155,20 @@ A repository can be public or private. A private repository is only visible to y >>> update_repo_visibility(repo_id=repo_id, private=True) ``` +### Update repository settings (Gated Access) + +The `update_repo_settings` function allows you to control the gated access feature of a repository. Gated access restricts access to the repository's files, requiring users to request access before they can view or download the content. +You can update the settings of a repository, using the `update_repo_settings` function as shown in the following: + +**Note:** This method is currently designed to work primarily with **dataset** repositories. + +```py +>>> from huggingface_hub import HfApi + +>>> api = HfApi() +>>> api.update_repo_settings(repo_id=repo_id, gated="auto") # Set automatic gating for a dataset +``` + ### Rename your repository You can rename your repository on the Hub using [`move_repo`]. Using this method, you can also move the repo from a user to diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py index 9d699b57fb..5f9bc38f2d 100644 --- a/src/huggingface_hub/__init__.py +++ b/src/huggingface_hub/__init__.py @@ -249,10 +249,10 @@ "update_collection_metadata", "update_inference_endpoint", "update_repo_visibility", + "update_repo_settings", "update_webhook", "upload_file", "upload_folder", - "upload_large_folder", "whoami", ], "hf_file_system": [ @@ -754,10 +754,10 @@ def __dir__(): update_collection_metadata, # noqa: F401 update_inference_endpoint, # noqa: F401 update_repo_visibility, # noqa: F401 + update_repo_settings, #noqa: F401 update_webhook, # noqa: F401 upload_file, # noqa: F401 upload_folder, # noqa: F401 - upload_large_folder, # noqa: F401 whoami, # noqa: F401 ) from .hf_file_system import ( diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index b25e3ee717..0fe35c9dc9 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -75,7 +75,6 @@ plan_multi_commits, ) from ._space_api import SpaceHardware, SpaceRuntime, SpaceStorage, SpaceVariable -from ._upload_large_folder import upload_large_folder_internal from .community import ( Discussion, DiscussionComment, @@ -3542,6 +3541,39 @@ def update_repo_visibility( hf_raise_for_status(r) return r.json() + @validate_hf_hub_args + def update_repo_settings( + self, + repo_id: str, + gated: Union[str, bool] = False, + *, + token: Union[str, bool, None] = None, + repo_type: Optional[str] = None, + ) -> Dict[str, Union[str, bool]]: + + if gated not in ["auto", "manual", False]: + raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False") + # Build headers + #headers = build_hf_headers(token=token) + r = get_session().put( + url=f"{self.endpoint}/api/datasets/{repo_id}/settings", + #url=f"https://huggingface.co/api/{repo_type}s/{repo_id}/settings", + headers=self._build_hf_headers(token=token), + json={"gated": gated}, + ) + + try: + hf_raise_for_status(r) + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + raise ValueError("Invalid or missing Hugging Face token. Please check your authentication.") from e + elif e.response.status_code == 404: + raise ValueError(f"Repository not found: {repo_id}") from e + else: + print(f"HTTP Error {e.response.status_code}: {e}") + + return r.json() + def move_repo( self, from_id: str, @@ -5173,123 +5205,6 @@ def delete_folder( parent_commit=parent_commit, ) - def upload_large_folder( - self, - repo_id: str, - folder_path: Union[str, Path], - *, - repo_type: str, # Repo type is required! - revision: Optional[str] = None, - private: bool = False, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - num_workers: Optional[int] = None, - print_report: bool = True, - print_report_every: int = 60, - ) -> None: - """Upload a large folder to the Hub in the most resilient way possible. - - Several workers are started to upload files in an optimized way. Before being committed to a repo, files must be - hashed and be pre-uploaded if they are LFS files. Workers will perform these tasks for each file in the folder. - At each step, some metadata information about the upload process is saved in the folder under `.cache/.huggingface/` - to be able to resume the process if interrupted. The whole process might result in several commits. - - Args: - repo_id (`str`): - The repository to which the file will be uploaded. - E.g. `"HuggingFaceTB/smollm-corpus"`. - folder_path (`str` or `Path`): - Path to the folder to upload on the local file system. - repo_type (`str`): - Type of the repository. Must be one of `"model"`, `"dataset"` or `"space"`. - Unlike in all other `HfApi` methods, `repo_type` is explicitly required here. This is to avoid - any mistake when uploading a large folder to the Hub, and therefore prevent from having to re-upload - everything. - revision (`str`, `optional`): - The branch to commit to. If not provided, the `main` branch will be used. - private (`bool`, `optional`): - Whether the repository should be private. Defaults to False. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are uploaded. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not uploaded. - num_workers (`int`, *optional*): - Number of workers to start. Defaults to `os.cpu_count() - 2` (minimum 2). - A higher number of workers may speed up the process if your machine allows it. However, on machines with a - slower connection, it is recommended to keep the number of workers low to ensure better resumability. - Indeed, partially uploaded files will have to be completely re-uploaded if the process is interrupted. - print_report (`bool`, *optional*): - Whether to print a report of the upload progress. Defaults to True. - Report is printed to `sys.stdout` every X seconds (60 by defaults) and overwrites the previous report. - print_report_every (`int`, *optional*): - Frequency at which the report is printed. Defaults to 60 seconds. - - - - A few things to keep in mind: - - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations - - Do not start several processes in parallel. - - You can interrupt and resume the process at any time. - - Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first. - - - - - - While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice: - - you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally. - - you cannot set a custom `commit_message` and `commit_description` since multiple commits are created. - - you cannot delete from the repo while uploading. Please make a separate commit first. - - you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`. - - - - **Technical details:** - - `upload_large_folder` process is as follow: - 1. (Check parameters and setup.) - 2. Create repo if missing. - 3. List local files to upload. - 4. Start workers. Workers can perform the following tasks: - - Hash a file. - - Get upload mode (regular or LFS) for a list of files. - - Pre-upload an LFS file. - - Commit a bunch of files. - Once a worker finishes a task, it will move on to the next task based on the priority list (see below) until - all files are uploaded and committed. - 5. While workers are up, regularly print a report to sys.stdout. - - Order of priority: - 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file). - 2. Commit if at least 25 files are ready to commit. - 3. Get upload mode if at least 10 files have been hashed. - 4. Pre-upload LFS file if at least 1 file and no worker is pre-uploading. - 5. Hash file if at least 1 file and no worker is hashing. - 6. Get upload mode if at least 1 file and no worker is getting upload mode. - 7. Pre-upload LFS file if at least 1 file (exception: if hf_transfer is enabled, only 1 worker can preupload LFS at a time). - 8. Hash file if at least 1 file to hash. - 9. Get upload mode if at least 1 file to get upload mode. - 10. Commit if at least 1 file to commit. - - Special rules: - - If `hf_transfer` is enabled, only 1 LFS uploader at a time. Otherwise the CPU would be bloated by `hf_transfer`. - - Only one worker can commit at a time. - - If no tasks are available, the worker waits for 10 seconds before checking again. - """ - return upload_large_folder_internal( - self, - repo_id=repo_id, - folder_path=folder_path, - repo_type=repo_type, - revision=revision, - private=private, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - num_workers=num_workers, - print_report=print_report, - print_report_every=print_report_every, - ) - @validate_hf_hub_args def get_hf_file_metadata( self, @@ -7673,6 +7588,7 @@ def create_inference_endpoint( "revision": revision, "task": task, "image": image, + "secrets": secrets, }, "name": name, "provider": { @@ -7681,8 +7597,7 @@ def create_inference_endpoint( }, "type": type, } - if secrets: - payload["model"]["secrets"] = secrets + response = get_session().post( f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", headers=self._build_hf_headers(token=token), @@ -9574,6 +9489,7 @@ def _parse_revision_from_pr_url(pr_url: str) -> str: create_repo = api.create_repo delete_repo = api.delete_repo update_repo_visibility = api.update_repo_visibility +update_repo_settings = api.update_repo_settings super_squash_history = api.super_squash_history move_repo = api.move_repo upload_file = api.upload_file @@ -9582,7 +9498,6 @@ def _parse_revision_from_pr_url(pr_url: str) -> str: delete_folder = api.delete_folder delete_files = api.delete_files create_commits_on_pr = api.create_commits_on_pr -upload_large_folder = api.upload_large_folder preupload_lfs_files = api.preupload_lfs_files create_branch = api.create_branch delete_branch = api.delete_branch diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index b5d551c0c2..e1a4101a1c 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -139,6 +139,7 @@ def test_repo_id_no_warning(): with warnings.catch_warnings(record=True) as record: repo_id = api.create_repo(repo_name()).repo_id api.update_repo_visibility(repo_id, private=True) + api.update_repo_settings(repo_id, gated="auto") api.delete_repo(repo_id) assert not len(record) @@ -223,6 +224,15 @@ def test_create_update_and_delete_repo(self): assert res["private"] res = self._api.update_repo_visibility(repo_id=repo_id, private=False) assert not res["private"] + + # Test gated status update (new functionality) + res = self._api.update_repo_settings(repo_id=repo_id, gated="auto") + assert res["gated"] == "auto" + res = self._api.update_repo_settings(repo_id=repo_id, gated="manual") + assert res["gated"] == "manual" + res = self._api.update_repo_settings(repo_id=repo_id, gated=False) + assert res["gated"] is False + self._api.delete_repo(repo_id=repo_id) def test_create_update_and_delete_model_repo(self): @@ -288,6 +298,18 @@ def test_move_repo_invalid_repo_id(self) -> None: with pytest.raises(ValueError, match=r"Invalid repo_id*"): self._api.move_repo(from_id="invalid_repo_id", to_id="namespace/repo_name") + ## Test for #2447 + ## See https://github.com/huggingface/huggingface_hub/issues/2447 + + #def test_update_repo_settings(self): + # repo_id = self._api.create_repo(repo_id=repo_name()).repo_id + # res = self._api.update_repo_settings(repo_id=repo_id, gated="auto") + # assert res["gated"] == "auto" + # res = self._api.update_repo_settings(repo_id=repo_id, gated="manual") + # assert res["gated"] == "manual" + # res = self._api.update_repo_settings(repo_id=repo_id, gated=False) + # assert res.get("gated") is False + # self._api.delete_repo(repo_id=repo_id) class CommitApiTest(HfApiCommonTest): def setUp(self) -> None: From 409baa5d431e55ad20708c1f76c7e26e78735147 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 3 Sep 2024 15:09:53 +0200 Subject: [PATCH 02/12] resolve merge conflicts --- src/huggingface_hub/__init__.py | 6 +- src/huggingface_hub/hf_api.py | 113 ++++++++++++++++++++++++++++++-- tests/test_hf_api.py | 9 +-- 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py index 5f9bc38f2d..6e76a13314 100644 --- a/src/huggingface_hub/__init__.py +++ b/src/huggingface_hub/__init__.py @@ -248,11 +248,12 @@ "update_collection_item", "update_collection_metadata", "update_inference_endpoint", - "update_repo_visibility", "update_repo_settings", + "update_repo_visibility", "update_webhook", "upload_file", "upload_folder", + "upload_large_folder", "whoami", ], "hf_file_system": [ @@ -753,11 +754,12 @@ def __dir__(): update_collection_item, # noqa: F401 update_collection_metadata, # noqa: F401 update_inference_endpoint, # noqa: F401 + update_repo_settings, # noqa: F401 update_repo_visibility, # noqa: F401 - update_repo_settings, #noqa: F401 update_webhook, # noqa: F401 upload_file, # noqa: F401 upload_folder, # noqa: F401 + upload_large_folder, # noqa: F401 whoami, # noqa: F401 ) from .hf_file_system import ( diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 048653cb5b..74395c7158 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -75,6 +75,7 @@ plan_multi_commits, ) from ._space_api import SpaceHardware, SpaceRuntime, SpaceStorage, SpaceVariable +from ._upload_large_folder import upload_large_folder_internal from .community import ( Discussion, DiscussionComment, @@ -3550,14 +3551,13 @@ def update_repo_settings( token: Union[str, bool, None] = None, repo_type: Optional[str] = None, ) -> Dict[str, Union[str, bool]]: - if gated not in ["auto", "manual", False]: - raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False") + raise ValueError("Invalid gated status, must be one of 'auto', 'manual', or False") # Build headers - #headers = build_hf_headers(token=token) + # headers = build_hf_headers(token=token) r = get_session().put( url=f"{self.endpoint}/api/datasets/{repo_id}/settings", - #url=f"https://huggingface.co/api/{repo_type}s/{repo_id}/settings", + # url=f"https://huggingface.co/api/{repo_type}s/{repo_id}/settings", headers=self._build_hf_headers(token=token), json={"gated": gated}, ) @@ -5205,6 +5205,111 @@ def delete_folder( parent_commit=parent_commit, ) + def upload_large_folder( + self, + repo_id: str, + folder_path: Union[str, Path], + *, + repo_type: str, # Repo type is required! + revision: Optional[str] = None, + private: bool = False, + allow_patterns: Optional[Union[List[str], str]] = None, + ignore_patterns: Optional[Union[List[str], str]] = None, + num_workers: Optional[int] = None, + print_report: bool = True, + print_report_every: int = 60, + ) -> None: + """Upload a large folder to the Hub in the most resilient way possible. + Several workers are started to upload files in an optimized way. Before being committed to a repo, files must be + hashed and be pre-uploaded if they are LFS files. Workers will perform these tasks for each file in the folder. + At each step, some metadata information about the upload process is saved in the folder under `.cache/.huggingface/` + to be able to resume the process if interrupted. The whole process might result in several commits. + Args: + repo_id (`str`): + The repository to which the file will be uploaded. + E.g. `"HuggingFaceTB/smollm-corpus"`. + folder_path (`str` or `Path`): + Path to the folder to upload on the local file system. + repo_type (`str`): + Type of the repository. Must be one of `"model"`, `"dataset"` or `"space"`. + Unlike in all other `HfApi` methods, `repo_type` is explicitly required here. This is to avoid + any mistake when uploading a large folder to the Hub, and therefore prevent from having to re-upload + everything. + revision (`str`, `optional`): + The branch to commit to. If not provided, the `main` branch will be used. + private (`bool`, `optional`): + Whether the repository should be private. Defaults to False. + allow_patterns (`List[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are uploaded. + ignore_patterns (`List[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not uploaded. + num_workers (`int`, *optional*): + Number of workers to start. Defaults to `os.cpu_count() - 2` (minimum 2). + A higher number of workers may speed up the process if your machine allows it. However, on machines with a + slower connection, it is recommended to keep the number of workers low to ensure better resumability. + Indeed, partially uploaded files will have to be completely re-uploaded if the process is interrupted. + print_report (`bool`, *optional*): + Whether to print a report of the upload progress. Defaults to True. + Report is printed to `sys.stdout` every X seconds (60 by defaults) and overwrites the previous report. + print_report_every (`int`, *optional*): + Frequency at which the report is printed. Defaults to 60 seconds. + + A few things to keep in mind: + - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations + - Do not start several processes in parallel. + - You can interrupt and resume the process at any time. + - Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first. + + + While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice: + - you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally. + - you cannot set a custom `commit_message` and `commit_description` since multiple commits are created. + - you cannot delete from the repo while uploading. Please make a separate commit first. + - you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`. + + **Technical details:** + `upload_large_folder` process is as follow: + 1. (Check parameters and setup.) + 2. Create repo if missing. + 3. List local files to upload. + 4. Start workers. Workers can perform the following tasks: + - Hash a file. + - Get upload mode (regular or LFS) for a list of files. + - Pre-upload an LFS file. + - Commit a bunch of files. + Once a worker finishes a task, it will move on to the next task based on the priority list (see below) until + all files are uploaded and committed. + 5. While workers are up, regularly print a report to sys.stdout. + Order of priority: + 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file). + 2. Commit if at least 25 files are ready to commit. + 3. Get upload mode if at least 10 files have been hashed. + 4. Pre-upload LFS file if at least 1 file and no worker is pre-uploading. + 5. Hash file if at least 1 file and no worker is hashing. + 6. Get upload mode if at least 1 file and no worker is getting upload mode. + 7. Pre-upload LFS file if at least 1 file (exception: if hf_transfer is enabled, only 1 worker can preupload LFS at a time). + 8. Hash file if at least 1 file to hash. + 9. Get upload mode if at least 1 file to get upload mode. + 10. Commit if at least 1 file to commit. + Special rules: + - If `hf_transfer` is enabled, only 1 LFS uploader at a time. Otherwise the CPU would be bloated by `hf_transfer`. + - Only one worker can commit at a time. + - If no tasks are available, the worker waits for 10 seconds before checking again. + """ + return upload_large_folder_internal( + self, + repo_id=repo_id, + folder_path=folder_path, + repo_type=repo_type, + revision=revision, + private=private, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + num_workers=num_workers, + print_report=print_report, + print_report_every=print_report_every, + ) + @validate_hf_hub_args def get_hf_file_metadata( self, diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index e1a4101a1c..c55f19e04c 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -224,7 +224,7 @@ def test_create_update_and_delete_repo(self): assert res["private"] res = self._api.update_repo_visibility(repo_id=repo_id, private=False) assert not res["private"] - + # Test gated status update (new functionality) res = self._api.update_repo_settings(repo_id=repo_id, gated="auto") assert res["gated"] == "auto" @@ -232,7 +232,7 @@ def test_create_update_and_delete_repo(self): assert res["gated"] == "manual" res = self._api.update_repo_settings(repo_id=repo_id, gated=False) assert res["gated"] is False - + self._api.delete_repo(repo_id=repo_id) def test_create_update_and_delete_model_repo(self): @@ -300,8 +300,8 @@ def test_move_repo_invalid_repo_id(self) -> None: ## Test for #2447 ## See https://github.com/huggingface/huggingface_hub/issues/2447 - - #def test_update_repo_settings(self): + + # def test_update_repo_settings(self): # repo_id = self._api.create_repo(repo_id=repo_name()).repo_id # res = self._api.update_repo_settings(repo_id=repo_id, gated="auto") # assert res["gated"] == "auto" @@ -311,6 +311,7 @@ def test_move_repo_invalid_repo_id(self) -> None: # assert res.get("gated") is False # self._api.delete_repo(repo_id=repo_id) + class CommitApiTest(HfApiCommonTest): def setUp(self) -> None: super().setUp() From f773f77e50c6fb86136380e74638901098c70560 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 3 Sep 2024 15:21:12 +0200 Subject: [PATCH 03/12] fix merge issue --- src/huggingface_hub/hf_api.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 74395c7158..2974881408 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -5220,10 +5220,12 @@ def upload_large_folder( print_report_every: int = 60, ) -> None: """Upload a large folder to the Hub in the most resilient way possible. + Several workers are started to upload files in an optimized way. Before being committed to a repo, files must be hashed and be pre-uploaded if they are LFS files. Workers will perform these tasks for each file in the folder. At each step, some metadata information about the upload process is saved in the folder under `.cache/.huggingface/` to be able to resume the process if interrupted. The whole process might result in several commits. + Args: repo_id (`str`): The repository to which the file will be uploaded. @@ -5253,6 +5255,7 @@ def upload_large_folder( Report is printed to `sys.stdout` every X seconds (60 by defaults) and overwrites the previous report. print_report_every (`int`, *optional*): Frequency at which the report is printed. Defaults to 60 seconds. + A few things to keep in mind: - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations @@ -5260,6 +5263,7 @@ def upload_large_folder( - You can interrupt and resume the process at any time. - Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first. + While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice: - you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally. @@ -5267,7 +5271,9 @@ def upload_large_folder( - you cannot delete from the repo while uploading. Please make a separate commit first. - you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`. + **Technical details:** + `upload_large_folder` process is as follow: 1. (Check parameters and setup.) 2. Create repo if missing. @@ -5280,6 +5286,7 @@ def upload_large_folder( Once a worker finishes a task, it will move on to the next task based on the priority list (see below) until all files are uploaded and committed. 5. While workers are up, regularly print a report to sys.stdout. + Order of priority: 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file). 2. Commit if at least 25 files are ready to commit. @@ -5291,6 +5298,7 @@ def upload_large_folder( 8. Hash file if at least 1 file to hash. 9. Get upload mode if at least 1 file to get upload mode. 10. Commit if at least 1 file to commit. + Special rules: - If `hf_transfer` is enabled, only 1 LFS uploader at a time. Otherwise the CPU would be bloated by `hf_transfer`. - Only one worker can commit at a time. @@ -7693,7 +7701,6 @@ def create_inference_endpoint( "revision": revision, "task": task, "image": image, - "secrets": secrets, }, "name": name, "provider": { @@ -7702,6 +7709,8 @@ def create_inference_endpoint( }, "type": type, } + if secrets: + payload["model"]["secrets"] = secrets response = get_session().post( f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", From 92c9531f42b55bdbfaa1c2d0f5bd756d35bb72e7 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 3 Sep 2024 15:22:29 +0200 Subject: [PATCH 04/12] merge issues --- src/huggingface_hub/hf_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 2974881408..1fbaf90c66 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -5257,19 +5257,23 @@ def upload_large_folder( Frequency at which the report is printed. Defaults to 60 seconds. + A few things to keep in mind: - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations - Do not start several processes in parallel. - You can interrupt and resume the process at any time. - Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first. + + While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice: - you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally. - you cannot set a custom `commit_message` and `commit_description` since multiple commits are created. - you cannot delete from the repo while uploading. Please make a separate commit first. - you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`. + **Technical details:** @@ -7711,7 +7715,6 @@ def create_inference_endpoint( } if secrets: payload["model"]["secrets"] = secrets - response = get_session().post( f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", headers=self._build_hf_headers(token=token), @@ -9612,6 +9615,7 @@ def _parse_revision_from_pr_url(pr_url: str) -> str: delete_folder = api.delete_folder delete_files = api.delete_files create_commits_on_pr = api.create_commits_on_pr +upload_large_folder = api.upload_large_folder preupload_lfs_files = api.preupload_lfs_files create_branch = api.create_branch delete_branch = api.delete_branch From cf3c78cbc3fc00dbb3903711da3cf7e7389a61dd Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Wed, 4 Sep 2024 03:03:27 +0530 Subject: [PATCH 05/12] Add `update_repo_settings` function to HfApi --- src/huggingface_hub/hf_api.py | 30 +++++------- tests/test_hf_api.py | 91 ++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 39 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 1fbaf90c66..2a51d64739 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -3546,32 +3546,28 @@ def update_repo_visibility( def update_repo_settings( self, repo_id: str, - gated: Union[str, bool] = False, + gated: Literal["auto", "manual", False] = False, *, token: Union[str, bool, None] = None, repo_type: Optional[str] = None, ) -> Dict[str, Union[str, bool]]: if gated not in ["auto", "manual", False]: - raise ValueError("Invalid gated status, must be one of 'auto', 'manual', or False") + raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.") + + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL # default repo type + # Build headers - # headers = build_hf_headers(token=token) + headers = self._build_hf_headers(token=token) + r = get_session().put( - url=f"{self.endpoint}/api/datasets/{repo_id}/settings", - # url=f"https://huggingface.co/api/{repo_type}s/{repo_id}/settings", - headers=self._build_hf_headers(token=token), + url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings", + headers=headers, json={"gated": gated}, ) - - try: - hf_raise_for_status(r) - except requests.exceptions.HTTPError as e: - if e.response.status_code == 401: - raise ValueError("Invalid or missing Hugging Face token. Please check your authentication.") from e - elif e.response.status_code == 404: - raise ValueError(f"Repository not found: {repo_id}") from e - else: - print(f"HTTP Error {e.response.status_code}: {e}") - + hf_raise_for_status(r) return r.json() def move_repo( diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index c55f19e04c..19edecb1a4 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -139,7 +139,6 @@ def test_repo_id_no_warning(): with warnings.catch_warnings(record=True) as record: repo_id = api.create_repo(repo_name()).repo_id api.update_repo_visibility(repo_id, private=True) - api.update_repo_settings(repo_id, gated="auto") api.delete_repo(repo_id) assert not len(record) @@ -224,15 +223,6 @@ def test_create_update_and_delete_repo(self): assert res["private"] res = self._api.update_repo_visibility(repo_id=repo_id, private=False) assert not res["private"] - - # Test gated status update (new functionality) - res = self._api.update_repo_settings(repo_id=repo_id, gated="auto") - assert res["gated"] == "auto" - res = self._api.update_repo_settings(repo_id=repo_id, gated="manual") - assert res["gated"] == "manual" - res = self._api.update_repo_settings(repo_id=repo_id, gated=False) - assert res["gated"] is False - self._api.delete_repo(repo_id=repo_id) def test_create_update_and_delete_model_repo(self): @@ -298,18 +288,75 @@ def test_move_repo_invalid_repo_id(self) -> None: with pytest.raises(ValueError, match=r"Invalid repo_id*"): self._api.move_repo(from_id="invalid_repo_id", to_id="namespace/repo_name") - ## Test for #2447 - ## See https://github.com/huggingface/huggingface_hub/issues/2447 - - # def test_update_repo_settings(self): - # repo_id = self._api.create_repo(repo_id=repo_name()).repo_id - # res = self._api.update_repo_settings(repo_id=repo_id, gated="auto") - # assert res["gated"] == "auto" - # res = self._api.update_repo_settings(repo_id=repo_id, gated="manual") - # assert res["gated"] == "manual" - # res = self._api.update_repo_settings(repo_id=repo_id, gated=False) - # assert res.get("gated") is False - # self._api.delete_repo(repo_id=repo_id) + def test_update_repo_settings(self): + repo_id = self._api.create_repo(repo_id=repo_name()).repo_id + # Test setting gated to "auto" + self._api.update_repo_settings(repo_id=repo_id, gated="auto") + res = self._api.model_info(repo_id, expand="gated") + assert res.gated == "auto" + # Test setting gated to "manual" + self._api.update_repo_settings(repo_id=repo_id, gated="manual") + res = self._api.model_info(repo_id, expand="gated") + assert res.gated == "manual" + # Test disabling gating + self._api.update_repo_settings(repo_id=repo_id, gated=False) + res = self._api.model_info(repo_id, expand="gated") + assert res.gated is False + self._api.delete_repo(repo_id=repo_id) + + def test_update_dataset_repo_settings(self): + repo_id = self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_DATASET).repo_id + # Test setting gated to "auto" + self._api.update_repo_settings(repo_id=repo_id, gated="auto", repo_type=constants.REPO_TYPE_DATASET) + res = self._api.dataset_info(repo_id, expand="gated") + assert res.gated == "auto" + # Test setting gated to "manual" + self._api.update_repo_settings(repo_id=repo_id, gated="manual", repo_type=constants.REPO_TYPE_DATASET) + res = self._api.dataset_info(repo_id, expand="gated") + assert res.gated == "manual" + # Test disabling gating + self._api.update_repo_settings(repo_id=repo_id, gated=False, repo_type=constants.REPO_TYPE_DATASET) + res = self._api.dataset_info(repo_id, expand="gated") + assert res.gated is False + self._api.delete_repo(repo_id=repo_id, repo_type=constants.REPO_TYPE_DATASET) + + def test_update_model_repo_settings(self): + repo_id = self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_MODEL).repo_id + # Test setting gated to "auto" + self._api.update_repo_settings(repo_id=repo_id, gated="auto", repo_type=constants.REPO_TYPE_MODEL) + res = self._api.model_info(repo_id, expand="gated") + assert res.gated == "auto" + # Test setting gated to "manual" + self._api.update_repo_settings(repo_id=repo_id, gated="manual", repo_type=constants.REPO_TYPE_MODEL) + res = self._api.model_info(repo_id, expand="gated") + assert res.gated == "manual" + # Test disabling gating + self._api.update_repo_settings(repo_id=repo_id, gated=False, repo_type=constants.REPO_TYPE_MODEL) + res = self._api.model_info(repo_id, expand="gated") + assert res.gated is False + self._api.delete_repo(repo_id=repo_id, repo_type=constants.REPO_TYPE_MODEL) + + def test_update_space_repo_settings(self): + with pytest.raises(ValueError, match=r"No space_sdk provided.*"): + self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_SPACE, space_sdk=None) + with pytest.raises(ValueError, match=r"Invalid space_sdk.*"): + self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_SPACE, space_sdk="something") + + for sdk in constants.SPACES_SDK_TYPES: + repo_id = self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_SPACE, space_sdk=sdk).repo_id + # Test setting gated to "auto" + self._api.update_repo_settings(repo_id=repo_id, gated="auto", repo_type=constants.REPO_TYPE_SPACE) + res = self._api.space_info(repo_id, expand="gated") + assert res.gated == "auto" + # Test setting gated to "manual" + self._api.update_repo_settings(repo_id=repo_id, gated="manual", repo_type=constants.REPO_TYPE_SPACE) + res = self._api.space_info(repo_id, expand="gated") + assert res.gated == "manual" + # Test disabling gating + self._api.update_repo_settings(repo_id=repo_id, gated=False, repo_type=constants.REPO_TYPE_SPACE) + res = self._api.space_info(repo_id, expand="gated") + assert res.gated is False + self._api.delete_repo(repo_id=repo_id, repo_type=constants.REPO_TYPE_SPACE) class CommitApiTest(HfApiCommonTest): From 25448985c705722301b3060ebe2a8da446904ccb Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:16:09 +0530 Subject: [PATCH 06/12] Add `update_repo_settings` function to HfApi --- src/huggingface_hub/hf_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 7868005875..12ae87473a 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -3546,8 +3546,8 @@ def update_repo_visibility( def update_repo_settings( self, repo_id: str, - gated: Literal["auto", "manual", False] = False, *, + gated: Literal["auto", "manual", False] = False, token: Union[str, bool, None] = None, repo_type: Optional[str] = None, ) -> Dict[str, Union[str, bool]]: @@ -3568,7 +3568,6 @@ def update_repo_settings( json={"gated": gated}, ) hf_raise_for_status(r) - return r.json() def move_repo( self, From 3e507a9c6484db65c2be68d0583263dd64759b3c Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:51:58 +0530 Subject: [PATCH 07/12] Enhance HfApi with `update_repo_settings` function --- docs/source/en/guides/repository.md | 9 ++- tests/test_hf_api.py | 89 ++++++++--------------------- 2 files changed, 27 insertions(+), 71 deletions(-) diff --git a/docs/source/en/guides/repository.md b/docs/source/en/guides/repository.md index 733255868e..77bf797e61 100644 --- a/docs/source/en/guides/repository.md +++ b/docs/source/en/guides/repository.md @@ -155,18 +155,17 @@ A repository can be public or private. A private repository is only visible to y >>> update_repo_visibility(repo_id=repo_id, private=True) ``` -### Update repository settings (Gated Access) +### Setup gated access -The `update_repo_settings` function allows you to control the gated access feature of a repository. Gated access restricts access to the repository's files, requiring users to request access before they can view or download the content. -You can update the settings of a repository, using the `update_repo_settings` function as shown in the following: +To give more control over how repos are used, the Hub allows repo authors to enable **access requests** for their repos. User must agree to share their contact information (username and email address) with the repo authors to access the files when enabled. A repo with access requests enabled is called a **gated repo**. -**Note:** This method is currently designed to work primarily with **dataset** repositories. +You can set a repo as gated using [`update_repo_settings`]: ```py >>> from huggingface_hub import HfApi >>> api = HfApi() ->>> api.update_repo_settings(repo_id=repo_id, gated="auto") # Set automatic gating for a dataset +>>> api.update_repo_settings(repo_id=repo_id, gated="auto") # Set automatic gating for a model ``` ### Rename your repository diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index 3f55674c50..d4ea145079 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -288,75 +288,32 @@ def test_move_repo_invalid_repo_id(self) -> None: with pytest.raises(ValueError, match=r"Invalid repo_id*"): self._api.move_repo(from_id="invalid_repo_id", to_id="namespace/repo_name") - def test_update_repo_settings(self): - repo_id = self._api.create_repo(repo_id=repo_name()).repo_id - # Test setting gated to "auto" - self._api.update_repo_settings(repo_id=repo_id, gated="auto") - res = self._api.model_info(repo_id, expand="gated") - assert res.gated == "auto" - # Test setting gated to "manual" - self._api.update_repo_settings(repo_id=repo_id, gated="manual") - res = self._api.model_info(repo_id, expand="gated") - assert res.gated == "manual" - # Test disabling gating - self._api.update_repo_settings(repo_id=repo_id, gated=False) - res = self._api.model_info(repo_id, expand="gated") - assert res.gated is False - self._api.delete_repo(repo_id=repo_id) - - def test_update_dataset_repo_settings(self): - repo_id = self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_DATASET).repo_id - # Test setting gated to "auto" - self._api.update_repo_settings(repo_id=repo_id, gated="auto", repo_type=constants.REPO_TYPE_DATASET) - res = self._api.dataset_info(repo_id, expand="gated") - assert res.gated == "auto" - # Test setting gated to "manual" - self._api.update_repo_settings(repo_id=repo_id, gated="manual", repo_type=constants.REPO_TYPE_DATASET) - res = self._api.dataset_info(repo_id, expand="gated") - assert res.gated == "manual" - # Test disabling gating - self._api.update_repo_settings(repo_id=repo_id, gated=False, repo_type=constants.REPO_TYPE_DATASET) - res = self._api.dataset_info(repo_id, expand="gated") - assert res.gated is False - self._api.delete_repo(repo_id=repo_id, repo_type=constants.REPO_TYPE_DATASET) + @use_tmp_repo(repo_type="model") + def test_update_repo_settings(self, repo_url: RepoUrl): + repo_id = repo_url.repo_id + + for gated_value in ["auto", "manual", False]: + self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) + info = self._api.model_info(repo_id, expand="gated") + assert info.gated == gated_value - def test_update_model_repo_settings(self): - repo_id = self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_MODEL).repo_id - # Test setting gated to "auto" - self._api.update_repo_settings(repo_id=repo_id, gated="auto", repo_type=constants.REPO_TYPE_MODEL) - res = self._api.model_info(repo_id, expand="gated") - assert res.gated == "auto" - # Test setting gated to "manual" - self._api.update_repo_settings(repo_id=repo_id, gated="manual", repo_type=constants.REPO_TYPE_MODEL) - res = self._api.model_info(repo_id, expand="gated") - assert res.gated == "manual" - # Test disabling gating - self._api.update_repo_settings(repo_id=repo_id, gated=False, repo_type=constants.REPO_TYPE_MODEL) - res = self._api.model_info(repo_id, expand="gated") - assert res.gated is False - self._api.delete_repo(repo_id=repo_id, repo_type=constants.REPO_TYPE_MODEL) + @use_tmp_repo(repo_type="dataset") + def test_update_dataset_repo_settings(self, repo_url: RepoUrl): + repo_id = repo_url.repo_id + + for gated_value in ["auto", "manual", False]: + self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) + info = self._api.dataset_info(repo_id, expand="gated") + assert info.gated == gated_value - def test_update_space_repo_settings(self): - with pytest.raises(ValueError, match=r"No space_sdk provided.*"): - self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_SPACE, space_sdk=None) - with pytest.raises(ValueError, match=r"Invalid space_sdk.*"): - self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_SPACE, space_sdk="something") + @use_tmp_repo(repo_type="space") + def test_update_space_repo_settings(self, repo_url: RepoUrl): + repo_id = repo_url.repo_id - for sdk in constants.SPACES_SDK_TYPES: - repo_id = self._api.create_repo(repo_id=repo_name(), repo_type=constants.REPO_TYPE_SPACE, space_sdk=sdk).repo_id - # Test setting gated to "auto" - self._api.update_repo_settings(repo_id=repo_id, gated="auto", repo_type=constants.REPO_TYPE_SPACE) - res = self._api.space_info(repo_id, expand="gated") - assert res.gated == "auto" - # Test setting gated to "manual" - self._api.update_repo_settings(repo_id=repo_id, gated="manual", repo_type=constants.REPO_TYPE_SPACE) - res = self._api.space_info(repo_id, expand="gated") - assert res.gated == "manual" - # Test disabling gating - self._api.update_repo_settings(repo_id=repo_id, gated=False, repo_type=constants.REPO_TYPE_SPACE) - res = self._api.space_info(repo_id, expand="gated") - assert res.gated is False - self._api.delete_repo(repo_id=repo_id, repo_type=constants.REPO_TYPE_SPACE) + for gated_value in ["auto", "manual", False]: + self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) + info = self._api.space_info(repo_id, expand="gated") + assert info.gated == gated_value class CommitApiTest(HfApiCommonTest): From 8e6f9cfaa8f66fdfd252119d2998471985f3be73 Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Tue, 10 Sep 2024 21:36:32 +0530 Subject: [PATCH 08/12] Enhance HfApi with `update_repo_settings` function --- src/huggingface_hub/hf_api.py | 6 +++--- tests/test_hf_api.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index ef3ef6cf7b..16291e8dd7 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -3585,15 +3585,15 @@ def update_repo_settings( ) -> Dict[str, Union[str, bool]]: if gated not in ["auto", "manual", False]: raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.") - + if repo_type not in constants.REPO_TYPES: raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") if repo_type is None: repo_type = constants.REPO_TYPE_MODEL # default repo type - + # Build headers headers = self._build_hf_headers(token=token) - + r = get_session().put( url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings", headers=headers, diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index d940056cf4..e9b86ed312 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -291,7 +291,7 @@ def test_move_repo_invalid_repo_id(self) -> None: @use_tmp_repo(repo_type="model") def test_update_repo_settings(self, repo_url: RepoUrl): repo_id = repo_url.repo_id - + for gated_value in ["auto", "manual", False]: self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) info = self._api.model_info(repo_id, expand="gated") @@ -300,7 +300,7 @@ def test_update_repo_settings(self, repo_url: RepoUrl): @use_tmp_repo(repo_type="dataset") def test_update_dataset_repo_settings(self, repo_url: RepoUrl): repo_id = repo_url.repo_id - + for gated_value in ["auto", "manual", False]: self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) info = self._api.dataset_info(repo_id, expand="gated") @@ -309,7 +309,7 @@ def test_update_dataset_repo_settings(self, repo_url: RepoUrl): @use_tmp_repo(repo_type="space") def test_update_space_repo_settings(self, repo_url: RepoUrl): repo_id = repo_url.repo_id - + for gated_value in ["auto", "manual", False]: self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) info = self._api.space_info(repo_id, expand="gated") From 59c05a58c965c862759d83fa137281e11dc5c5e7 Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Tue, 10 Sep 2024 22:20:45 +0530 Subject: [PATCH 09/12] Enhance HfApi with `update_repo_settings` function --- src/huggingface_hub/hf_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 16291e8dd7..5ce18b4dad 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -3582,7 +3582,7 @@ def update_repo_settings( gated: Literal["auto", "manual", False] = False, token: Union[str, bool, None] = None, repo_type: Optional[str] = None, - ) -> Dict[str, Union[str, bool]]: + ) -> None: if gated not in ["auto", "manual", False]: raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.") From 2bd4767b07c861a101e34c551ef6da8274cbed21 Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Wed, 11 Sep 2024 21:55:48 +0530 Subject: [PATCH 10/12] Enhance HfApi with `update_repo_settings` function --- tests/test_hf_api.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py index 467f2be734..44e228871f 100644 --- a/tests/test_hf_api.py +++ b/tests/test_hf_api.py @@ -293,21 +293,13 @@ def test_update_repo_settings(self, repo_url: RepoUrl): @use_tmp_repo(repo_type="dataset") def test_update_dataset_repo_settings(self, repo_url: RepoUrl): repo_id = repo_url.repo_id + repo_type = repo_url.repo_type for gated_value in ["auto", "manual", False]: - self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) + self._api.update_repo_settings(repo_id=repo_id, repo_type=repo_type, gated=gated_value) info = self._api.dataset_info(repo_id, expand="gated") assert info.gated == gated_value - @use_tmp_repo(repo_type="space") - def test_update_space_repo_settings(self, repo_url: RepoUrl): - repo_id = repo_url.repo_id - - for gated_value in ["auto", "manual", False]: - self._api.update_repo_settings(repo_id=repo_id, gated=gated_value) - info = self._api.space_info(repo_id, expand="gated") - assert info.gated == gated_value - class CommitApiTest(HfApiCommonTest): def setUp(self) -> None: From fbe8cbe118df9795d47d8b689d87cceaa829315b Mon Sep 17 00:00:00 2001 From: Swapnil Jikar <112884653+WizKnight@users.noreply.github.com> Date: Wed, 11 Sep 2024 22:49:45 +0530 Subject: [PATCH 11/12] Enhance HfApi with `update_repo_settings` function --- src/huggingface_hub/hf_api.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index fb47102a58..89a165ec38 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -3573,6 +3573,37 @@ def update_repo_settings( token: Union[str, bool, None] = None, repo_type: Optional[str] = None, ) -> None: + """ + Update the gated settings of a repository. + To give more control over how repos are used, the Hub allows repo authors to enable **access requests** for their repos. + + Args: + repo_id (str): + A namespace (user or an organization) and a repo name separated by a /. + gated (Literal["auto", "manual", False], optional, defaults to False): + The gated release status for the repository. + * "auto": The repository is gated, and access requests are automatically approved or denied based on predefined criteria. + * "manual": The repository is gated, and access requests require manual approval. + * False: The repository is not gated, and anyone can access it. + token (Union[str, bool, None], optional): + A valid user access token (string). Defaults to the locally saved token, + which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass False. + repo_type (str, optional): + The type of repository. + Must be one of the values in constants.REPO_TYPES. + Set to `"dataset"` if uploading to a dataset, `None` or `"model"` if uploading to a model. + Default is `"model"`. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If gated is not one of "auto", "manual", or False. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If repo_type is not one of the values in constants.REPO_TYPES. + [`~utils.HfHubHTTPError`]: + If the request to the Hugging Face Hub API fails. + """ if gated not in ["auto", "manual", False]: raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.") From ef61fbb0f6dd9f5359016f0d1549ac6b77361b22 Mon Sep 17 00:00:00 2001 From: Lucain Date: Thu, 12 Sep 2024 18:10:04 +0200 Subject: [PATCH 12/12] Apply suggestions from code review --- src/huggingface_hub/hf_api.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 89a165ec38..50643fa626 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -3578,23 +3578,21 @@ def update_repo_settings( To give more control over how repos are used, the Hub allows repo authors to enable **access requests** for their repos. Args: - repo_id (str): + repo_id (`str`): A namespace (user or an organization) and a repo name separated by a /. - gated (Literal["auto", "manual", False], optional, defaults to False): + gated (`Literal["auto", "manual", False]`, *optional*): The gated release status for the repository. * "auto": The repository is gated, and access requests are automatically approved or denied based on predefined criteria. * "manual": The repository is gated, and access requests require manual approval. - * False: The repository is not gated, and anyone can access it. - token (Union[str, bool, None], optional): + * False (default): The repository is not gated, and anyone can access it. + token (`Union[str, bool, None]`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass False. - repo_type (str, optional): - The type of repository. - Must be one of the values in constants.REPO_TYPES. - Set to `"dataset"` if uploading to a dataset, `None` or `"model"` if uploading to a model. - Default is `"model"`. + repo_type (`str`, *optional*): + The type of the repository to update settings from (`"model"`, `"dataset"` or `"space"`. + Defaults to `"model"`. Raises: [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)