diff --git a/fsspec/spec.py b/fsspec/spec.py index a39671605..b2fd144e4 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -2,14 +2,20 @@ import logging import os import warnings -from distutils.version import LooseVersion from errno import ESPIPE from hashlib import sha256 +from distutils.version import LooseVersion from glob import has_magic from .dircache import DirCache from .transaction import Transaction -from .utils import read_block, tokenize, stringify_path, other_paths +from .utils import ( + read_block, + tokenize, + stringify_path, + other_paths, + get_package_version_without_import, +) from .config import apply_config logger = logging.getLogger("fsspec") @@ -72,16 +78,13 @@ def __call__(cls, *args, **kwargs): return obj -try: # optionally derive from pyarrow's FileSystem, if available +pa_version = get_package_version_without_import("pyarrow") +if pa_version and LooseVersion(pa_version) < LooseVersion("2.0"): import pyarrow as pa -except ImportError: - up = object + + up = pa.filesystem.DaskFileSystem else: - # only derive from the legacy pyarrow's FileSystem for older pyarrow versions - if LooseVersion(pa.__version__) < LooseVersion("2.0"): - up = pa.filesystem.DaskFileSystem - else: - up = object + up = object class AbstractFileSystem(up, metaclass=_Cached): diff --git a/fsspec/utils.py b/fsspec/utils.py index 78df77e70..e2efee5b1 100644 --- a/fsspec/utils.py +++ b/fsspec/utils.py @@ -3,6 +3,7 @@ import os import pathlib import re +import sys from urllib.parse import urlsplit @@ -383,10 +384,13 @@ def can_be_local(path): return False -def setup_logger(logname, level="DEBUG"): +def setup_logger(logname, level="DEBUG", clear=True): + """Add standard logging handler to logger of given name""" import logging logger = logging.getLogger(logname) + if clear: + logger.handlers.clear() handle = logging.StreamHandler() formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s " "- %(message)s" @@ -395,3 +399,39 @@ def setup_logger(logname, level="DEBUG"): logger.addHandler(handle) logger.setLevel(level) return logger + + +def get_package_version_without_import(name): + """For given package name, try to find the version without importing it + + Import and package.__version__ is still the backup here, so an import + *might* happen. + + Returns either the version string, or None if the package + or the version was not readily found. + """ + if name in sys.modules: + mod = sys.modules[name] + if hasattr(mod, "__version__"): + return mod.__version__ + if sys.version_info >= (3, 8): + try: + import importlib.metadata + + return importlib.metadata.distribution(name).version + except ImportError: + pass + else: + try: + import importlib_metadata + + return importlib_metadata.distribution(name).version + except ImportError: + pass + try: + import importlib + + mod = importlib.import_module(name) + return mod.__version__ + except (ImportError, AttributeError): + return None