From 5f0dbd4af9fd2fcd7042329807f872399221c2bc Mon Sep 17 00:00:00 2001 From: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> Date: Mon, 16 Dec 2024 18:57:32 +0100 Subject: [PATCH 1/2] add hf:// --- etils/epath/gpath.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etils/epath/gpath.py b/etils/epath/gpath.py index 02a8e869..cec2e1b3 100644 --- a/etils/epath/gpath.py +++ b/etils/epath/gpath.py @@ -36,13 +36,14 @@ _P = TypeVar('_P') -_URI_PREFIXES = ('gs://', 's3://', 'az://') -_URI_SCHEMES = frozenset(('gs', 's3', 'az')) +_URI_PREFIXES = ('gs://', 's3://', 'az://', 'hf://') +_URI_SCHEMES = frozenset(('gs', 's3', 'az', 'hf')) _URI_MAP_ROOT = { 'gs://': '/gs/', 's3://': '/s3/', 'az://': '/az/', + 'hf://': '/hf/', } @@ -58,6 +59,7 @@ def _epath_use_tf() -> bool: 'gs': backend_lib.fsspec_backend, 's3': backend_lib.fsspec_backend, 'az': backend_lib.fsspec_backend, + 'hf': backend_lib.fsspec_backend, None: backend_lib.os_backend, } _GCS_BACKENDS = frozenset({ @@ -71,7 +73,7 @@ def _epath_use_tf() -> bool: class _GPath(abstract_path.Path): - """Pathlib like api with gs://, s3://, az:// support.""" + """Pathlib like api with gs://, s3://, az://, hf:// support.""" # `_PATH` is `posixpath` or `ntpath`. # Use explicit `join()` rather than `super().joinpath()` to avoid infinite From 28380615816a5b069052a222516719473873d063 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:38:34 +0100 Subject: [PATCH 2/2] in backend.py too --- etils/epath/backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etils/epath/backend.py b/etils/epath/backend.py index 9c342183..d9cc7dd8 100644 --- a/etils/epath/backend.py +++ b/etils/epath/backend.py @@ -440,6 +440,8 @@ def fs(self, path: PathLike) -> fsspec.AbstractFileSystem: return self._get_filesystem('s3') elif path.startswith('az://'): return self._get_filesystem('az') + elif path.startswith('hf://'): + return self._get_filesystem('hf') else: return self._get_filesystem('file')