From ab7d29d7ec5fc685b704abc714129ead22dea2da Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 11 Sep 2024 17:52:58 +0200 Subject: [PATCH 1/2] Fix race-condition issue when downloading from multiple threads --- src/huggingface_hub/file_download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py index 363cf86f65..09a32aa583 100644 --- a/src/huggingface_hub/file_download.py +++ b/src/huggingface_hub/file_download.py @@ -1388,7 +1388,8 @@ def _hf_hub_download_to_cache_dir( filename=filename, force_download=force_download, ) - _create_symlink(blob_path, pointer_path, new_blob=True) + if not os.path.exists(blob_path): + _create_symlink(blob_path, pointer_path, new_blob=True) return pointer_path From 833898ea3178debd4db3c4a8e1b91e3ed52b0bab Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 11 Sep 2024 18:26:17 +0200 Subject: [PATCH 2/2] correct fix --- src/huggingface_hub/file_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py index 09a32aa583..2d60f9b6c1 100644 --- a/src/huggingface_hub/file_download.py +++ b/src/huggingface_hub/file_download.py @@ -1388,7 +1388,7 @@ def _hf_hub_download_to_cache_dir( filename=filename, force_download=force_download, ) - if not os.path.exists(blob_path): + if not os.path.exists(pointer_path): _create_symlink(blob_path, pointer_path, new_blob=True) return pointer_path