From 2c234499777a5d3f5a213fbfc42b289c207c411b Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Tue, 31 Jan 2023 17:03:48 +0100 Subject: [PATCH 1/4] Cache supported tags for wheels. This fixes https://github.com/pypa/setuptools/issues/3804 --- changelog.d/3804.change.rst | 1 + setuptools/tests/test_wheel.py | 1 + setuptools/wheel.py | 15 +++++++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelog.d/3804.change.rst diff --git a/changelog.d/3804.change.rst b/changelog.d/3804.change.rst new file mode 100644 index 0000000000..86a6597c23 --- /dev/null +++ b/changelog.d/3804.change.rst @@ -0,0 +1 @@ +Cache supported tags for wheels. diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py index b2bbdfae7f..8b2faff607 100644 --- a/setuptools/tests/test_wheel.py +++ b/setuptools/tests/test_wheel.py @@ -612,6 +612,7 @@ def sys_tags(): for t in parse_tag('cp36-cp36m-manylinux1_x86_64'): yield t monkeypatch.setattr('setuptools.wheel.sys_tags', sys_tags) + monkeypatch.setattr('setuptools.wheel._supported_tags', None) assert Wheel( 'onnxruntime-0.1.2-cp36-cp36m-manylinux1_x86_64.whl').is_compatible() diff --git a/setuptools/wheel.py b/setuptools/wheel.py index e388083ba8..aab7ed0541 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -27,6 +27,8 @@ NAMESPACE_PACKAGE_INIT = \ "__import__('pkg_resources').declare_namespace(__name__)\n" +_supported_tags = None + def unpack(src_dir, dst_dir): '''Move everything under `src_dir` to `dst_dir`, and delete the former.''' @@ -82,10 +84,15 @@ def tags(self): ) def is_compatible(self): - '''Is the wheel is compatible with the current platform?''' - supported_tags = set( - (t.interpreter, t.abi, t.platform) for t in sys_tags()) - return next((True for t in self.tags() if t in supported_tags), False) + '''Is the wheel compatible with the current platform?''' + global _supported_tags + if _supported_tags is None: + # We calculate the supported tags only once, otherwise calling + # this method on thousands of wheels takes seconds instead of + # milliseconds. + _supported_tags = set( + (t.interpreter, t.abi, t.platform) for t in sys_tags()) + return next((True for t in self.tags() if t in _supported_tags), False) def egg_name(self): return _egg_basename( From 8af6bd4818558e4e4427e730ede8bc7ba17ca000 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Tue, 7 Mar 2023 22:36:16 +0100 Subject: [PATCH 2/4] Use functools.lru_cache to cache supported tags for wheels. This is a suggestion by @abravalheri for my PR. https://github.com/pypa/setuptools/pull/3805#issuecomment-1434361907 --- setuptools/tests/test_wheel.py | 14 +++++++++++--- setuptools/wheel.py | 17 +++++++++-------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py index 8b2faff607..934cf7f3ce 100644 --- a/setuptools/tests/test_wheel.py +++ b/setuptools/tests/test_wheel.py @@ -612,9 +612,17 @@ def sys_tags(): for t in parse_tag('cp36-cp36m-manylinux1_x86_64'): yield t monkeypatch.setattr('setuptools.wheel.sys_tags', sys_tags) - monkeypatch.setattr('setuptools.wheel._supported_tags', None) - assert Wheel( - 'onnxruntime-0.1.2-cp36-cp36m-manylinux1_x86_64.whl').is_compatible() + # Clear the supported tags cache, otherwise the sys_tags monkeypatch + # has no effect. + setuptools.wheel._supported_tags.cache_clear() + try: + assert Wheel( + 'onnxruntime-0.1.2-cp36-cp36m-manylinux1_x86_64.whl' + ).is_compatible() + finally: + # Clear the cache again, otherwise the sys_tags monkeypatch + # is still in effect for the rest of the tests. + setuptools.wheel._supported_tags.cache_clear() def test_wheel_mode(): diff --git a/setuptools/wheel.py b/setuptools/wheel.py index aab7ed0541..ff29e2fb09 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -2,6 +2,7 @@ import email import itertools +import functools import os import posixpath import re @@ -27,7 +28,13 @@ NAMESPACE_PACKAGE_INIT = \ "__import__('pkg_resources').declare_namespace(__name__)\n" -_supported_tags = None + +@functools.lru_cache(maxsize=None) +def _get_supported_tags(): + # We calculate the supported tags only once, otherwise calling + # this method on thousands of wheels takes seconds instead of + # milliseconds. + return set((t.interpreter, t.abi, t.platform) for t in sys_tags()) def unpack(src_dir, dst_dir): @@ -85,13 +92,7 @@ def tags(self): def is_compatible(self): '''Is the wheel compatible with the current platform?''' - global _supported_tags - if _supported_tags is None: - # We calculate the supported tags only once, otherwise calling - # this method on thousands of wheels takes seconds instead of - # milliseconds. - _supported_tags = set( - (t.interpreter, t.abi, t.platform) for t in sys_tags()) + _supported_tags = _get_supported_tags() return next((True for t in self.tags() if t in _supported_tags), False) def egg_name(self): From 8961851931e0bb92c7d91411d6bdcf0c98ca9767 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 8 Mar 2023 00:07:13 +0000 Subject: [PATCH 3/4] Simplify changes targeting sys_tags cache --- setuptools/tests/test_wheel.py | 21 +++++++-------------- setuptools/wheel.py | 5 ++--- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py index 934cf7f3ce..559e16d758 100644 --- a/setuptools/tests/test_wheel.py +++ b/setuptools/tests/test_wheel.py @@ -609,20 +609,13 @@ def test_wheel_no_dist_dir(): def test_wheel_is_compatible(monkeypatch): def sys_tags(): - for t in parse_tag('cp36-cp36m-manylinux1_x86_64'): - yield t - monkeypatch.setattr('setuptools.wheel.sys_tags', sys_tags) - # Clear the supported tags cache, otherwise the sys_tags monkeypatch - # has no effect. - setuptools.wheel._supported_tags.cache_clear() - try: - assert Wheel( - 'onnxruntime-0.1.2-cp36-cp36m-manylinux1_x86_64.whl' - ).is_compatible() - finally: - # Clear the cache again, otherwise the sys_tags monkeypatch - # is still in effect for the rest of the tests. - setuptools.wheel._supported_tags.cache_clear() + return { + (t.interpreter, t.abi, t.platform) + for t in parse_tag('cp36-cp36m-manylinux1_x86_64') + } + monkeypatch.setattr('setuptools.wheel._get_supported_tags', sys_tags) + assert Wheel( + 'onnxruntime-0.1.2-cp36-cp36m-manylinux1_x86_64.whl').is_compatible() def test_wheel_mode(): diff --git a/setuptools/wheel.py b/setuptools/wheel.py index ff29e2fb09..850e43cd01 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -34,7 +34,7 @@ def _get_supported_tags(): # We calculate the supported tags only once, otherwise calling # this method on thousands of wheels takes seconds instead of # milliseconds. - return set((t.interpreter, t.abi, t.platform) for t in sys_tags()) + return {(t.interpreter, t.abi, t.platform) for t in sys_tags()} def unpack(src_dir, dst_dir): @@ -92,8 +92,7 @@ def tags(self): def is_compatible(self): '''Is the wheel compatible with the current platform?''' - _supported_tags = _get_supported_tags() - return next((True for t in self.tags() if t in _supported_tags), False) + return next((True for t in self.tags() if t in _get_supported_tags()), False) def egg_name(self): return _egg_basename( From f18f7adae298a7f43a60fb3c1813fe6ca9f4ccdd Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 8 Mar 2023 09:53:58 +0000 Subject: [PATCH 4/4] Change news fragment to misc The introduced caching can be considered an internal change and minor optimization. --- changelog.d/{3804.change.rst => 3804.misc.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{3804.change.rst => 3804.misc.rst} (100%) diff --git a/changelog.d/3804.change.rst b/changelog.d/3804.misc.rst similarity index 100% rename from changelog.d/3804.change.rst rename to changelog.d/3804.misc.rst