From fe40fb6dcace5e05205c370042dc4810d10c5ab6 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 5 Apr 2021 15:04:40 +0200 Subject: [PATCH 1/3] Replace pkg_resources with importlib_metadata Fixes #10964. The other reason for this change is that pkg_resources has been [deprecated](https://github.com/pypa/setuptools/commit/8fe85c22cee7fde5e6af571b30f864bad156a010) in favor of importlib_metadata. --- src/transformers/utils/versions.py | 21 ++++++++++++++------- tests/test_versions_utils.py | 16 +++++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/transformers/utils/versions.py b/src/transformers/utils/versions.py index eabd92e54fbd7d..028dbcc6c836a4 100644 --- a/src/transformers/utils/versions.py +++ b/src/transformers/utils/versions.py @@ -22,7 +22,12 @@ from packaging import version -import pkg_resources + +# The package importlib_metadata is in a different place, depending on the python version. +if sys.version_info < (3, 8): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata ops = { @@ -39,7 +44,7 @@ def require_version(requirement: str, hint: Optional[str] = None) -> None: """ Perform a runtime check of the dependency versions, using the exact same syntax used by pip. - The installed module version comes from the `site-packages` dir via `pkg_resources`. + The installed module version comes from the `site-packages` dir via `importlib_metadata`. Args: requirement (:obj:`str`): pip style definition, e.g., "tokenizers==0.9.4", "tqdm>=4.27", "numpy" @@ -70,20 +75,22 @@ def require_version(requirement: str, hint: Optional[str] = None) -> None: if pkg == "python": got_ver = ".".join([str(x) for x in sys.version_info[:3]]) if not ops[op](version.parse(got_ver), version.parse(want_ver)): - raise pkg_resources.VersionConflict( + raise ImportError( f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}." ) return # check if any version is installed try: - got_ver = pkg_resources.get_distribution(pkg).version - except pkg_resources.DistributionNotFound: - raise pkg_resources.DistributionNotFound(requirement, ["this application", hint]) + got_ver = importlib_metadata.version(pkg) + except importlib_metadata.PackageNotFoundError: + raise importlib_metadata.PackageNotFoundError( + f"The '{requirement}' distribution was not found and is required by this application. {hint}" + ) # check that the right version is installed if version number was provided if want_ver is not None and not ops[op](version.parse(got_ver), version.parse(want_ver)): - raise pkg_resources.VersionConflict( + raise ImportError( f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}" ) diff --git a/tests/test_versions_utils.py b/tests/test_versions_utils.py index 902192a3f3eaa8..b1f8da5d61e44e 100644 --- a/tests/test_versions_utils.py +++ b/tests/test_versions_utils.py @@ -16,7 +16,13 @@ import numpy -import pkg_resources + +# The package importlib_metadata is in a different place, depending on the python version. +if sys.version_info < (3, 8): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + from transformers.testing_utils import TestCasePlus from transformers.utils.versions import require_version, require_version_core, require_version_examples @@ -57,7 +63,7 @@ def test_core(self): for req in ["numpy==1.0.0", "numpy>=1000.0.0", f"numpy<{numpy_ver}"]: try: require_version_core(req) - except pkg_resources.VersionConflict as e: + except ImportError as e: self.assertIn(f"{req} is required", str(e)) self.assertIn("but found", str(e)) @@ -65,7 +71,7 @@ def test_core(self): for req in ["numpipypie>1", "numpipypie2"]: try: require_version_core(req) - except pkg_resources.DistributionNotFound as e: + except importlib_metadata.PackageNotFoundError as e: self.assertIn(f"The '{req}' distribution was not found and is required by this application", str(e)) self.assertIn("Try: pip install transformers -U", str(e)) @@ -87,7 +93,7 @@ def test_examples(self): # the main functionality is tested in `test_core`, this is just the hint check try: require_version_examples("numpy>1000.4.5") - except pkg_resources.VersionConflict as e: + except ImportError as e: self.assertIn("is required", str(e)) self.assertIn("pip install -r examples/requirements.txt", str(e)) @@ -100,6 +106,6 @@ def test_python(self): for req in ["python>9.9.9", "python<3.0.0"]: try: require_version_core(req) - except pkg_resources.VersionConflict as e: + except ImportError as e: self.assertIn(f"{req} is required", str(e)) self.assertIn(f"but found python=={python_ver}", str(e)) From c2d0d12a24bb7d867b4fbbd700629034078aaa21 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 5 Apr 2021 19:15:20 +0200 Subject: [PATCH 2/3] Reduce to a single importlib_metadata import switch --- src/transformers/file_utils.py | 8 +------- tests/test_versions_utils.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 24020ea8c7b6ae..ed4b84dc108da8 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -46,19 +46,13 @@ import requests from filelock import FileLock +from transformers.utils.versions import importlib_metadata from . import __version__ from .hf_api import HfFolder from .utils import logging -# The package importlib_metadata is in a different place, depending on the python version. -if sys.version_info < (3, 8): - import importlib_metadata -else: - import importlib.metadata as importlib_metadata - - logger = logging.get_logger(__name__) # pylint: disable=invalid-name ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} diff --git a/tests/test_versions_utils.py b/tests/test_versions_utils.py index b1f8da5d61e44e..04c6d78ec39d55 100644 --- a/tests/test_versions_utils.py +++ b/tests/test_versions_utils.py @@ -16,15 +16,13 @@ import numpy - -# The package importlib_metadata is in a different place, depending on the python version. -if sys.version_info < (3, 8): - import importlib_metadata -else: - import importlib.metadata as importlib_metadata - from transformers.testing_utils import TestCasePlus -from transformers.utils.versions import require_version, require_version_core, require_version_examples +from transformers.utils.versions import ( + importlib_metadata, + require_version, + require_version_core, + require_version_examples, +) numpy_ver = numpy.__version__ From 14fe34d86180c005167c909e3b9717da2db2f70e Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 5 Apr 2021 10:26:52 -0700 Subject: [PATCH 3/3] Trigger CI