From 5e54f8256a979c4e4e5473ede8f222c0e6acbc2c Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Fri, 13 Apr 2018 12:01:52 -0700 Subject: [PATCH 1/3] Add support for pex resolver blacklist --- .../backend/python/subsystems/python_setup.py | 15 +++++++++++++++ .../pants/backend/python/tasks/pex_build_util.py | 6 ++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index 5fd308ffa2c..d8cbabab2de 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -64,6 +64,17 @@ def register_options(cls, register): help='A list of paths to search for python interpreters. Note that if a PEX_PYTHON_PATH ' 'variable is defined in a pexrc file, those interpreter paths will take precedence over ' 'this option.') + register('--resolver-blacklist', advanced=True, type=dict, default={}, + metavar='', + help='A blacklist dict (str->str) that maps package name to an interpreter ' + 'constraint. If a package name is in the blacklist and its interpreter ' + 'constraint matches the target interpreter, skip the requirement. This is needed ' + 'to ensure that universal requirement resolves for a target interpreter version do ' + 'not error out on interpreter specific requirements such as backport libs like ' + '`functools32`. For example, a valid blacklist is {"functools32": "CPython>3"}. ' + 'NOTE: this keyword is a temporary fix and will be reverted per: ' + 'https://github.com/pantsbuild/pants/issues/5696. The long term ' + 'solution is tracked by: https://github.com/pantsbuild/pex/issues/456.') @property def interpreter_constraints(self): @@ -109,6 +120,10 @@ def resolver_cache_ttl(self): def resolver_allow_prereleases(self): return self.get_options().resolver_allow_prereleases + @property + def resolver_blacklist(self): + return self.get_options().resolver_blacklist + @property def artifact_cache_dir(self): """Note that this is unrelated to the general pants artifact cache.""" diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index bc7fe6b8bed..f8d173da750 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -162,7 +162,8 @@ def dump_requirements(builder, interpreter, reqs, log, platforms=None): find_links = OrderedSet() for req in deduped_reqs: log.debug(' Dumping requirement: {}'.format(req)) - builder.add_requirement(req.requirement) + if not req.key in PythonSetup.global_instance().resolver_blacklist: + builder.add_requirement(req.requirement) if req.repository: find_links.add(req.repository) @@ -209,6 +210,7 @@ def _resolve_multi(interpreter, requirements, platforms, find_links): context=python_repos.get_network_context(), cache=requirements_cache_dir, cache_ttl=python_setup.resolver_cache_ttl, - allow_prereleases=python_setup.resolver_allow_prereleases) + allow_prereleases=python_setup.resolver_allow_prereleases, + pkg_blacklist=python_setup.resolver_blacklist) return distributions From b53db243e1315b61977ceccac6044c154f4ab529 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Fri, 13 Apr 2018 13:49:41 -0700 Subject: [PATCH 2/3] Add tests and bump pex library version --- 3rdparty/python/requirements.txt | 2 +- .../resolver_blacklist_testing/BUILD | 18 +++++++++++++++ .../resolver_blacklist_testing/main.py | 3 +++ .../tasks/test_python_run_integration.py | 22 +++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD create mode 100644 testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index ca596c728f6..5fd09fa542f 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -14,7 +14,7 @@ packaging==16.8 pathspec==0.5.0 parameterized==0.6.1 pep8==1.6.2 -pex==1.3.1 +pex==1.3.2 psutil==4.3.0 pyflakes==1.1.0 Pygments==1.4 diff --git a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD new file mode 100644 index 00000000000..e0a66118a94 --- /dev/null +++ b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD @@ -0,0 +1,18 @@ +python_binary( + name='test_bin', + source='main.py', + compatibility=['CPython>3'], + dependencies=[ + ':reqlib' + ] +) + +# Without blacklisting, pex resolver will error out +# on a transitive requirement of jupyter (functools32) +# when resolving for Python 3. +python_requirement_library( + name='reqlib', + requirements=[ + python_requirement('jupyter') + ] +) diff --git a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py new file mode 100644 index 00000000000..ae24a33eeba --- /dev/null +++ b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/main.py @@ -0,0 +1,3 @@ +import jupyter +print(jupyter) +print('Successful.') diff --git a/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py index 5e5137cc1fa..2706eb708b0 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_run_integration.py @@ -184,3 +184,25 @@ def _run_echo_version(self, version): '--quiet'] pants_run = self.run_pants(command=command) return pants_run.stdout_data.rstrip().split('\n')[-1] + + def test_pex_resolver_blacklist_integration(self): + py3 = '3' + if self.skip_if_no_python(py3): + return + pex = os.path.join(os.getcwd(), 'dist', 'test_bin.pex') + try: + pants_ini_config = {'python-setup': {'resolver_blacklist': {"functools32": "CPython>3"}}} + # clean-all to ensure that Pants resolves requirements for each run. + pants_binary_36 = self.run_pants( + command=['clean-all', 'binary', '{}:test_bin'.format(os.path.join(self.testproject,'resolver_blacklist_testing'))], + config=pants_ini_config + ) + self.assert_success(pants_binary_36) + pants_run_36 = self.run_pants( + command=['clean-all', 'run', '{}:test_bin'.format(os.path.join(self.testproject,'resolver_blacklist_testing'))], + config=pants_ini_config + ) + self.assert_success(pants_run_36) + finally: + if os.path.exists(pex): + os.remove(pex) From b18e1e78f09558f75ef7a188bbdaaff30c7938a1 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Mon, 16 Apr 2018 10:23:52 -0700 Subject: [PATCH 3/3] Address @cosmicexplorer comment --- .../resolver_blacklist_testing/BUILD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD index e0a66118a94..15f041f0028 100644 --- a/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD +++ b/testprojects/src/python/interpreter_selection/resolver_blacklist_testing/BUILD @@ -11,8 +11,8 @@ python_binary( # on a transitive requirement of jupyter (functools32) # when resolving for Python 3. python_requirement_library( - name='reqlib', - requirements=[ - python_requirement('jupyter') - ] + name='reqlib', + requirements=[ + python_requirement('jupyter'), + ] )