From 1344ade9064b1f5fd0bdc0e307ee218dfb2580f8 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 9 Oct 2018 13:44:45 -0400 Subject: [PATCH] Upgrade to pex 1.5.0; ~kill --resolver-blacklist. PEX now handles blacklisting for us by respecting PEP-508 enviornment markers. Fixes #5696 --- 3rdparty/python/requirements.txt | 2 +- .../pants/backend/python/interpreter_cache.py | 24 +++++------ .../backend/python/subsystems/python_setup.py | 8 ++-- .../backend/python/tasks/pex_build_util.py | 8 ++-- src/python/pants/init/plugin_resolver.py | 20 +++++----- .../resolver_blacklist_testing/BUILD | 40 ------------------- .../import_futures.py | 5 --- .../resolver_blacklist_testing/main.py | 5 --- .../python/test_apache_thrift_py_gen.py | 10 ++--- .../tasks/test_python_run_integration.py | 26 ------------ .../backend/python/test_interpreter_cache.py | 6 +-- 11 files changed, 39 insertions(+), 115 deletions(-) delete mode 100644 testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD delete mode 100644 testprojects/src/python/interpreter_selection/resolver_blacklist_testing/import_futures.py delete mode 100644 testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 183268df2e3..27f9f4fafb0 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -14,7 +14,7 @@ mock==2.0.0 packaging==16.8 parameterized==0.6.1 pathspec==0.5.0 -pex==1.4.8 +pex==1.5.0 psutil==4.3.0 pycodestyle==2.4.0 pyflakes==2.0.0 diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py index 2a112b8f171..34c0139bb4b 100644 --- a/src/python/pants/backend/python/interpreter_cache.py +++ b/src/python/pants/backend/python/interpreter_cache.py @@ -221,22 +221,22 @@ def _resolve_and_link(self, interpreter, requirement, target_link): # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into # wheels. precedence = (EggPackage, SourcePackage) - distributions = resolve(requirements=[requirement], - fetchers=self._python_repos.get_fetchers(), - interpreter=interpreter, - # The local interpreter cache is, by definition, composed of - # interpreters for the 'current' platform. - platform='current', - context=self._python_repos.get_network_context(), - precedence=precedence) - if not distributions: + resolved_dists = resolve(requirements=[requirement], + fetchers=self._python_repos.get_fetchers(), + interpreter=interpreter, + # The local interpreter cache is, by definition, composed of + # interpreters for the 'current' platform. + platform='current', + context=self._python_repos.get_network_context(), + precedence=precedence) + if not resolved_dists: return None - assert len(distributions) == 1, ('Expected exactly 1 distribution to be resolved for {}, ' + assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, ' 'found:\n\t{}'.format(requirement, - '\n\t'.join(map(str, distributions)))) + '\n\t'.join(map(str, resolved_dists)))) - dist_location = distributions[0].location + dist_location = resolved_dists[0].distribution.location target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) shutil.move(dist_location, target_location) _safe_link(target_location, target_link) diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index c26164405bf..6e914c01e06 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -57,6 +57,10 @@ def register_options(cls, register): 'variable is defined in a pexrc file, those interpreter paths will take precedence over ' 'this option.') register('--resolver-blacklist', advanced=True, type=dict, default={}, + removal_version='1.13.0.dev2', + removal_hint='Now unused. PEX handles blacklisting automatically via PEP-508 ' + 'environment markers: ' + 'https://www.python.org/dev/peps/pep-0508/#environment-markers', metavar='', help='A blacklist dict (str->str) that maps package name to an interpreter ' 'constraint. If a package name is in the blacklist and its interpreter ' @@ -114,10 +118,6 @@ def resolver_cache_ttl(self): def resolver_allow_prereleases(self): return self.get_options().resolver_allow_prereleases - @property - def resolver_blacklist(self): - return self.get_options().resolver_blacklist - @property def use_manylinux(self): return self.get_options().resolver_use_manylinux diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index 4c739b202da..7442090023b 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -112,11 +112,9 @@ def dump_requirements(builder, interpreter, reqs, log, platforms=None): """ deduped_reqs = OrderedSet(reqs) find_links = OrderedSet() - blacklist = PythonSetup.global_instance().resolver_blacklist for req in deduped_reqs: log.debug(' Dumping requirement: {}'.format(req)) - if not (req.key in blacklist and interpreter.identity.matches(blacklist[req.key])): - builder.add_requirement(req.requirement) + builder.add_requirement(req.requirement) if req.repository: find_links.add(req.repository) @@ -155,7 +153,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links): for platform in platforms: requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir, str(interpreter.identity)) - distributions[platform] = resolve( + resolved_dists = resolve( requirements=[req.requirement for req in requirements], interpreter=interpreter, fetchers=fetchers, @@ -164,7 +162,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links): cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl, allow_prereleases=python_setup.resolver_allow_prereleases, - pkg_blacklist=python_setup.resolver_blacklist, use_manylinux=python_setup.use_manylinux) + distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists] return distributions diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index 3948f613ac6..2c3704429a2 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -108,15 +108,17 @@ def _resolve_exact_plugin_locations(self): def _resolve_plugins(self): logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements))) - return resolver.resolve(self._plugin_requirements, - fetchers=self._python_repos.get_fetchers(), - context=self._python_repos.get_network_context(), - cache=self.plugin_cache_dir, - cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire. - allow_prereleases=PANTS_SEMVER.is_prerelease, - # Plugins will all depend on `pantsbuild.pants` which is distributed as - # a manylinux wheel. - use_manylinux=True) + resolved_dists = resolver.resolve(self._plugin_requirements, + fetchers=self._python_repos.get_fetchers(), + context=self._python_repos.get_network_context(), + cache=self.plugin_cache_dir, + # Effectively never expire. + cache_ttl=10 * 365 * 24 * 60 * 60, + allow_prereleases=PANTS_SEMVER.is_prerelease, + # Plugins will all depend on `pantsbuild.pants` which is + # distributed as a manylinux wheel. + use_manylinux=True) + return [resolved_dist.distribution for resolved_dist in resolved_dists] @memoized_property def plugin_cache_dir(self): diff --git a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD deleted file mode 100644 index faa956ac5ba..00000000000 --- a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -python_binary( - name='test_bin', - source='main.py', - compatibility=['CPython>=3.6'], - dependencies=[ - ':reqlib' - ] -) - -# This library requires pex blacklisting under python 3, because the pex resolver will error out -# on a transitive requirement of jupyter (functools32). -# -# This test also runs under python 2 though, and so in order to avoid dep floats of jupyter's -# transitive deps (which can float upward to versions that do not support python 2), we pin them. -python_requirement_library( - name='reqlib', - requirements=[ - python_requirement('jupyter==1.0.0'), - python_requirement('jupyter_console<5.4'), - python_requirement('ipykernel<5'), - python_requirement('ipython<6'), - ] -) - -# Test non-blacklisted backport usage. -python_binary( - name='test_py2', - source='import_futures.py', - compatibility=['CPython<3'], - dependencies=[ - ':futures_reqlib' - ] -) - -python_requirement_library( - name='futures_reqlib', - requirements=[ - python_requirement('futures'), - ] -) diff --git a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/import_futures.py b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/import_futures.py deleted file mode 100644 index 4a18df85db8..00000000000 --- a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/import_futures.py +++ /dev/null @@ -1,5 +0,0 @@ -from concurrent.futures import Future - - -print(Future) -print('Successful.') diff --git a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py deleted file mode 100644 index bcca7b5e19e..00000000000 --- a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py +++ /dev/null @@ -1,5 +0,0 @@ -import jupyter - - -print(jupyter) -print('Successful.') diff --git a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py index 14596bd6691..24ac9b71c9b 100644 --- a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py +++ b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py @@ -156,11 +156,11 @@ def test_namespace_effective(self): # https://github.com/pantsbuild/pants/issues/5975 pythonpath = list(interpreter.extras.values()) pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets) - for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))], - interpreter=interpreter, - context=python_repos.get_network_context(), - fetchers=python_repos.get_fetchers()): - pythonpath.append(dist.location) + for resolved_dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))], + interpreter=interpreter, + context=python_repos.get_network_context(), + fetchers=python_repos.get_fetchers()): + pythonpath.append(resolved_dist.distribution.location) process = subprocess.Popen([interpreter.binary, '-c', diff --git a/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py index 56727ce0957..d1658f9f921 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py @@ -200,29 +200,3 @@ def test_target_constraints_with_no_sources(self): self.assertIn('CPython>3', py2_info.interpreter_constraints) # Cleanup. os.remove(py2_pex) - - @skip_unless_python36 - def test_pex_resolver_blacklist_integration(self): - pex = os.path.join(os.getcwd(), 'dist', 'test_bin.pex') - try: - pants_ini_config = {'python-setup': {'resolver_blacklist': {'functools32': 'CPython>3'}}} - target_address_base = os.path.join(self.testproject, 'resolver_blacklist_testing') - # clean-all to ensure that Pants resolves requirements for each run. - pants_binary_36 = self.run_pants( - command=['clean-all', 'binary', '{}:test_bin'.format(target_address_base)], - config=pants_ini_config - ) - self.assert_success(pants_binary_36) - pants_run_36 = self.run_pants( - command=['clean-all', 'run', '{}:test_bin'.format(target_address_base)], - config=pants_ini_config - ) - self.assert_success(pants_run_36) - pants_run_27 = self.run_pants( - command=['clean-all', 'run', '{}:test_py2'.format(target_address_base)], - config=pants_ini_config - ) - self.assert_success(pants_run_27) - finally: - if os.path.exists(pex): - os.remove(pex) diff --git a/tests/python/pants_test/backend/python/test_interpreter_cache.py b/tests/python/pants_test/backend/python/test_interpreter_cache.py index 870f1b7c1ee..68902505950 100644 --- a/tests/python/pants_test/backend/python/test_interpreter_cache.py +++ b/tests/python/pants_test/backend/python/test_interpreter_cache.py @@ -85,11 +85,11 @@ def link_egg(repo_root, requirement): existing_dist = Package.from_href(existing_dist_location) requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version) - distributions = resolve([requirement], + resolved_dists = resolve([requirement], interpreter=self._interpreter, precedence=(EggPackage, SourcePackage)) - self.assertEqual(1, len(distributions)) - dist_location = distributions[0].location + self.assertEqual(1, len(resolved_dists)) + dist_location = resolved_dists[0].distribution.location self.assertRegexpMatches(dist_location, r'\.egg$') os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))