Skip to content

Commit

Permalink
[HfFileSystem] Faster fs.walk() (huggingface#2346)
Browse files Browse the repository at this point in the history
* faster fs walk

* add missing args
  • Loading branch information
lhoestq authored and MoritzLaurer committed Jul 3, 2024
1 parent c282c95 commit 7832db5
Showing 1 changed file with 6 additions and 0 deletions.
6 changes: 6 additions & 0 deletions src/huggingface_hub/hf_file_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,12 @@ def _ls_tree(
out.append(cache_path_info)
return out

def walk(self, path, *args, **kwargs):
# Set expand_info=False by default to get a x10 speed boost
kwargs = {"expand_info": kwargs.get("detail", False), **kwargs}
path = self.resolve_path(path, revision=kwargs.get("revision")).unresolve()
yield from super().walk(path, *args, **kwargs)

def glob(self, path, **kwargs):
# Set expand_info=False by default to get a x10 speed boost
kwargs = {"expand_info": kwargs.get("detail", False), **kwargs}
Expand Down

0 comments on commit 7832db5

Please sign in to comment.