diff --git a/setup.cfg b/setup.cfg index bf310953..f3a3cfe4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,7 @@ dev = # pyarrow pydantic pydantic-settings + smbprotocol [options.package_data] upath = diff --git a/upath/_flavour_sources.py b/upath/_flavour_sources.py index ab22e010..f9af2b72 100644 --- a/upath/_flavour_sources.py +++ b/upath/_flavour_sources.py @@ -17,6 +17,7 @@ without a direct dependency on the underlying filesystem package. """ + # # skipping protocols: # - blockcache @@ -66,10 +67,10 @@ def __init_subclass__(cls: Any, **kwargs): class AbstractFileSystemFlavour(FileSystemFlavourBase): - __orig_class__ = 'fsspec.spec.AbstractFileSystem' - __orig_version__ = '2024.2.0' - protocol = 'abstract' - root_marker = '' + __orig_class__ = "fsspec.spec.AbstractFileSystem" + __orig_version__ = "2024.2.0" + protocol = "abstract" + root_marker = "" @classmethod def _strip_protocol(cls, path): @@ -114,11 +115,11 @@ def _parent(cls, path): class AsyncLocalFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'morefs.asyn_local.AsyncLocalFileSystem' - __orig_version__ = '0.2.0' + __orig_class__ = "morefs.asyn_local.AsyncLocalFileSystem" + __orig_version__ = "0.2.0" protocol = () - root_marker = '/' - sep = '/' + root_marker = "/" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -143,11 +144,11 @@ def _parent(cls, path): class AzureBlobFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'adlfs.spec.AzureBlobFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('abfs', 'az', 'abfss') - root_marker = '' - sep = '/' + __orig_class__ = "adlfs.spec.AzureBlobFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("abfs", "az", "abfss") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path: str): @@ -216,11 +217,11 @@ def _get_kwargs_from_urls(urlpath): class AzureDatalakeFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'adlfs.gen1.AzureDatalakeFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('adl',) - root_marker = '' - sep = '/' + __orig_class__ = "adlfs.gen1.AzureDatalakeFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("adl",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -238,11 +239,11 @@ def _get_kwargs_from_urls(paths): class BoxFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'boxfs.boxfs.BoxFileSystem' - __orig_version__ = '0.2.1' - protocol = ('box',) - root_marker = '' - sep = '/' + __orig_class__ = "boxfs.boxfs.BoxFileSystem" + __orig_version__ = "0.2.1" + protocol = ("box",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path) -> str: @@ -252,11 +253,11 @@ def _strip_protocol(cls, path) -> str: class DaskWorkerFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.dask.DaskWorkerFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('dask',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.dask.DaskWorkerFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("dask",) + root_marker = "" + sep = "/" @staticmethod def _get_kwargs_from_urls(path): @@ -268,27 +269,27 @@ def _get_kwargs_from_urls(path): class DataFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.data.DataFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('data',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.data.DataFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("data",) + root_marker = "" + sep = "/" class DatabricksFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.dbfs.DatabricksFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('dbfs',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.dbfs.DatabricksFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("dbfs",) + root_marker = "" + sep = "/" class DictFSFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'morefs.dict.DictFS' - __orig_version__ = '0.2.0' - protocol = ('dictfs',) - root_marker = '' - sep = '/' + __orig_class__ = "morefs.dict.DictFS" + __orig_version__ = "0.2.0" + protocol = ("dictfs",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path: str) -> str: @@ -301,19 +302,19 @@ def _strip_protocol(cls, path: str) -> str: class DropboxDriveFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'dropboxdrivefs.core.DropboxDriveFileSystem' - __orig_version__ = '1.3.1' - protocol = ('dropbox',) - root_marker = '' - sep = '/' + __orig_class__ = "dropboxdrivefs.core.DropboxDriveFileSystem" + __orig_version__ = "1.3.1" + protocol = ("dropbox",) + root_marker = "" + sep = "/" class FTPFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.ftp.FTPFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('ftp',) - root_marker = '/' - sep = '/' + __orig_class__ = "fsspec.implementations.ftp.FTPFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("ftp",) + root_marker = "/" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -328,11 +329,11 @@ def _get_kwargs_from_urls(urlpath): class GCSFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'gcsfs.core.GCSFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('gcs', 'gs') - root_marker = '' - sep = '/' + __orig_class__ = "gcsfs.core.GCSFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("gcs", "gs") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -405,11 +406,11 @@ def _split_path(cls, path, version_aware=False): class GitFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.git.GitFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('git',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.git.GitFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("git",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -433,11 +434,11 @@ def _get_kwargs_from_urls(path): class GithubFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.github.GithubFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('github',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.github.GithubFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("github",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -458,11 +459,11 @@ def _get_kwargs_from_urls(path): class HTTPFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.http.HTTPFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('http', 'https') - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.http.HTTPFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("http", "https") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -479,11 +480,11 @@ def _parent(cls, path): class HadoopFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.arrow.HadoopFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('hdfs', 'arrow_hdfs') - root_marker = '/' - sep = '/' + __orig_class__ = "fsspec.implementations.arrow.HadoopFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("hdfs", "arrow_hdfs") + root_marker = "/" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -512,27 +513,27 @@ def _get_kwargs_from_urls(path): class HfFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'huggingface_hub.hf_file_system.HfFileSystem' - __orig_version__ = '0.20.3' - protocol = ('hf',) - root_marker = '' - sep = '/' + __orig_class__ = "huggingface_hub.hf_file_system.HfFileSystem" + __orig_version__ = "0.20.3" + protocol = ("hf",) + root_marker = "" + sep = "/" class JupyterFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.jupyter.JupyterFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('jupyter', 'jlab') - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.jupyter.JupyterFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("jupyter", "jlab") + root_marker = "" + sep = "/" class LakeFSFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'lakefs_spec.spec.LakeFSFileSystem' - __orig_version__ = '0.7.0' - protocol = ('lakefs',) - root_marker = '' - sep = '/' + __orig_class__ = "lakefs_spec.spec.LakeFSFileSystem" + __orig_version__ = "0.7.0" + protocol = ("lakefs",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -546,11 +547,11 @@ def _strip_protocol(cls, path): class LibArchiveFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.libarchive.LibArchiveFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('libarchive',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.libarchive.LibArchiveFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("libarchive",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -559,11 +560,11 @@ def _strip_protocol(cls, path): class LocalFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.local.LocalFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('file', 'local') - root_marker = '/' - sep = '/' + __orig_class__ = "fsspec.implementations.local.LocalFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("file", "local") + root_marker = "/" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -588,25 +589,27 @@ def _parent(cls, path): class MemFSFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'morefs.memory.MemFS' - __orig_version__ = '0.2.0' - protocol = ('memfs',) - root_marker = '' - sep = '/' + __orig_class__ = "morefs.memory.MemFS" + __orig_version__ = "0.2.0" + protocol = ("memfs",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): if path.startswith("memfs://"): path = path[len("memfs://") :] - return MemoryFileSystemFlavour._strip_protocol(path) # pylint: disable=protected-access + return MemoryFileSystemFlavour._strip_protocol( + path + ) # pylint: disable=protected-access class MemoryFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.memory.MemoryFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('memory',) - root_marker = '/' - sep = '/' + __orig_class__ = "fsspec.implementations.memory.MemoryFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("memory",) + root_marker = "/" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -619,11 +622,11 @@ def _strip_protocol(cls, path): class OCIFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'ocifs.core.OCIFileSystem' - __orig_version__ = '1.3.1' - protocol = ('oci', 'ocilake') - root_marker = '' - sep = '/' + __orig_class__ = "ocifs.core.OCIFileSystem" + __orig_version__ = "1.3.1" + protocol = ("oci", "ocilake") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -647,11 +650,11 @@ def _parent(cls, path): class OSSFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'ossfs.core.OSSFileSystem' - __orig_version__ = '2023.12.0' - protocol = ('oss',) - root_marker = '' - sep = '/' + __orig_class__ = "ossfs.core.OSSFileSystem" + __orig_version__ = "2023.12.0" + protocol = ("oss",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -687,27 +690,27 @@ def _strip_protocol(cls, path): class OverlayFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'morefs.overlay.OverlayFileSystem' - __orig_version__ = '0.2.0' - protocol = ('overlayfs',) - root_marker = '' - sep = '/' + __orig_class__ = "morefs.overlay.OverlayFileSystem" + __orig_version__ = "0.2.0" + protocol = ("overlayfs",) + root_marker = "" + sep = "/" class ReferenceFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.reference.ReferenceFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('reference',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.reference.ReferenceFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("reference",) + root_marker = "" + sep = "/" class S3FileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 's3fs.core.S3FileSystem' - __orig_version__ = '2024.2.0' - protocol = ('s3', 's3a') - root_marker = '' - sep = '/' + __orig_class__ = "s3fs.core.S3FileSystem" + __orig_version__ = "2024.2.0" + protocol = ("s3", "s3a") + root_marker = "" + sep = "/" @staticmethod def _get_kwargs_from_urls(urlpath): @@ -730,11 +733,11 @@ def _get_kwargs_from_urls(urlpath): class SFTPFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.sftp.SFTPFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('sftp', 'ssh') - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.sftp.SFTPFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("sftp", "ssh") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -749,11 +752,11 @@ def _get_kwargs_from_urls(urlpath): class SMBFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.smb.SMBFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('smb',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.smb.SMBFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("smb",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -769,27 +772,27 @@ def _get_kwargs_from_urls(path): class TarFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.tar.TarFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('tar',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.tar.TarFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("tar",) + root_marker = "" + sep = "/" class WandbFSFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'wandbfs._wandbfs.WandbFS' - __orig_version__ = '0.0.2' - protocol = ('wandb',) - root_marker = '' - sep = '/' + __orig_class__ = "wandbfs._wandbfs.WandbFS" + __orig_version__ = "0.0.2" + protocol = ("wandb",) + root_marker = "" + sep = "/" class WebHDFSFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.webhdfs.WebHDFS' - __orig_version__ = '2024.2.0' - protocol = ('webhdfs', 'webHDFS') - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.webhdfs.WebHDFS" + __orig_version__ = "2024.2.0" + protocol = ("webhdfs", "webHDFS") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -806,11 +809,11 @@ def _get_kwargs_from_urls(urlpath): class WebdavFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'webdav4.fsspec.WebdavFileSystem' - __orig_version__ = '0.9.8' - protocol = ('webdav', 'dav') - root_marker = '' - sep = '/' + __orig_class__ = "webdav4.fsspec.WebdavFileSystem" + __orig_version__ = "0.9.8" + protocol = ("webdav", "dav") + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path: str) -> str: @@ -820,11 +823,11 @@ def _strip_protocol(cls, path: str) -> str: class XRootDFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec_xrootd.xrootd.XRootDFileSystem' - __orig_version__ = '0.2.4' - protocol = ('root',) - root_marker = '/' - sep = '/' + __orig_class__ = "fsspec_xrootd.xrootd.XRootDFileSystem" + __orig_version__ = "0.2.4" + protocol = ("root",) + root_marker = "/" + sep = "/" @classmethod def _strip_protocol(cls, path: str | list[str]) -> Any: @@ -846,11 +849,11 @@ def _get_kwargs_from_urls(u: str) -> dict[Any, Any]: class ZipFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'fsspec.implementations.zip.ZipFileSystem' - __orig_version__ = '2024.2.0' - protocol = ('zip',) - root_marker = '' - sep = '/' + __orig_class__ = "fsspec.implementations.zip.ZipFileSystem" + __orig_version__ = "2024.2.0" + protocol = ("zip",) + root_marker = "" + sep = "/" @classmethod def _strip_protocol(cls, path): @@ -859,8 +862,8 @@ def _strip_protocol(cls, path): class _DVCFileSystemFlavour(AbstractFileSystemFlavour): - __orig_class__ = 'dvc.fs.dvc._DVCFileSystem' - __orig_version__ = '3.47.0' - protocol = ('dvc',) - root_marker = '/' - sep = '/' + __orig_class__ = "dvc.fs.dvc._DVCFileSystem" + __orig_version__ = "3.47.0" + protocol = ("dvc",) + root_marker = "/" + sep = "/" diff --git a/upath/implementations/smb.py b/upath/implementations/smb.py new file mode 100644 index 00000000..9bab14be --- /dev/null +++ b/upath/implementations/smb.py @@ -0,0 +1,32 @@ +import smbprotocol.exceptions + +from upath import UPath + + +class SMBPath(UPath): + __slots__ = () + + @property + def path(self): + return "/" + super().path + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + # smbclient does not support setting mode externally + if parents and not exist_ok and self.exists(): + raise FileExistsError(str(self)) + try: + self.fs.mkdir( + self.path, + create_parents=parents, + ) + except smbprotocol.exceptions.SMBOSError: + if not exist_ok: + raise FileExistsError(str(self)) + if not self.is_dir(): + raise FileExistsError(str(self)) + + def iterdir(self): + if not self.is_dir(): + raise NotADirectoryError(str(self)) + else: + return super().iterdir() diff --git a/upath/registry.py b/upath/registry.py index 7a54b7f3..c886e39c 100644 --- a/upath/registry.py +++ b/upath/registry.py @@ -80,6 +80,7 @@ class _Registry(MutableMapping[str, "type[upath.UPath]"]): "webdav+http": "upath.implementations.webdav.WebdavPath", "webdav+https": "upath.implementations.webdav.WebdavPath", "github": "upath.implementations.github.GitHubPath", + "smb": "upath.implementations.smb.SMBPath", } if TYPE_CHECKING: diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py index a2f85b0f..976623e5 100644 --- a/upath/tests/conftest.py +++ b/upath/tests/conftest.py @@ -12,6 +12,7 @@ import pytest from fsspec.implementations.local import LocalFileSystem from fsspec.implementations.local import make_path_posix +from fsspec.implementations.smb import SMBFileSystem from fsspec.registry import _registry from fsspec.registry import register_implementation from fsspec.utils import stringify_path @@ -409,3 +410,57 @@ def azure_fixture(azurite_credentials, azure_container): finally: for blob in client.list_blobs(): client.delete_blob(blob["name"]) + + +@pytest.fixture(scope="module") +def smb_container(): + try: + pchk = ["docker", "run", "--name", "fsspec_test_smb", "hello-world"] + subprocess.check_call(pchk) + stop_docker("fsspec_test_smb") + except (subprocess.CalledProcessError, FileNotFoundError): + pytest.skip("docker run not available") + + # requires docker + container = "fsspec_smb" + stop_docker(container) + cfg = "-p -u 'testuser;testpass' -s 'home;/share;no;no;no;testuser'" + port = 445 + img = f"docker run --name {container} --detach -p 139:139 -p {port}:445 dperson/samba" # noqa: E231 E501 + cmd = f"{img} {cfg}" + try: + subprocess.check_output(shlex.split(cmd)).strip().decode() + time.sleep(2) + yield { + "host": "localhost", + "port": port, + "username": "testuser", + "password": "testpass", + "register_session_retries": 100, # max ~= 10 seconds + } + finally: + import smbclient # pylint: disable=import-outside-toplevel + + smbclient.reset_connection_cache() + stop_docker(container) + + +@pytest.fixture +def smb_url(smb_container): + smb_url = "smb://{username}:{password}@{host}/home/" + smb_url = smb_url.format(**smb_container) + return smb_url + + +@pytest.fixture +def smb_fixture(local_testdir, smb_url, smb_container): + smb = SMBFileSystem( + host=smb_container["host"], + port=smb_container["port"], + username=smb_container["username"], + password=smb_container["password"], + ) + url = smb_url + "testdir/" + smb.put(local_testdir, "/home/testdir", recursive=True) + yield url + smb.delete("/home/testdir", recursive=True) diff --git a/upath/tests/implementations/test_smb.py b/upath/tests/implementations/test_smb.py new file mode 100644 index 00000000..e9d1fb92 --- /dev/null +++ b/upath/tests/implementations/test_smb.py @@ -0,0 +1,11 @@ +import pytest + +from upath import UPath +from upath.tests.cases import BaseTests + + +class TestUPathSMB(BaseTests): + + @pytest.fixture(autouse=True) + def path(self, smb_fixture): + self.path = UPath(smb_fixture)