From c031e05afbc19b62bd1bccc18b189d348383c538 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu, 12 Mar 2020 12:25:50 -0700 Subject: [PATCH] ipex additional fixes for PEX-INFO interpreter constraints add an integration test!! --- src/python/pants/python/BUILD | 30 ++++++++++++ src/python/pants/python/pex_build_util.py | 31 +++++++++---- .../python/pex_build_util_test_integration.py | 46 +++++++++++++++++++ .../python/test_apache_thrift_py_gen.py | 2 +- .../tasks/util/build_local_dists_test_base.py | 2 +- 5 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 src/python/pants/python/pex_build_util_test_integration.py diff --git a/src/python/pants/python/BUILD b/src/python/pants/python/BUILD index 29ee3849282..340c132fc34 100644 --- a/src/python/pants/python/BUILD +++ b/src/python/pants/python/BUILD @@ -15,3 +15,33 @@ python_library( 'src/python/pants/util:memo', ] ) + + +python_tests( + name='tests', + sources=['*_test.py', '!*_integration.py'], + dependencies=[ + '3rdparty/python:pex', + ':python', + 'src/python/pants/backend/python/targets', + 'src/python/pants/build_graph', + 'src/python/pants/util:contextutil', + 'src/python/pants/testutil/subsystem', + 'src/python/pants/testutil:pexrc_util', + 'src/python/pants/testutil:test_base', + ], + tags = {'partially_type_checked'}, +) + + +python_tests( + name='integration', + sources=['*_integration.py'], + dependencies=[ + '3rdparty/python:pex', + 'src/python/pants/util:contextutil', + 'src/python/pants/testutil:int-test', + 'testprojects/src/python:python_targets_directory', + ], + tags = {'integration', 'partially_type_checked'}, +) diff --git a/src/python/pants/python/pex_build_util.py b/src/python/pants/python/pex_build_util.py index 3dec873e35f..8ce170f1afa 100644 --- a/src/python/pants/python/pex_build_util.py +++ b/src/python/pants/python/pex_build_util.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple -from pex.fetcher import PyPIFetcher, Fetcher -from pex.interpreter import PythonInterpreter +from pex.fetcher import Fetcher, PyPIFetcher +from pex.interpreter import PythonIdentity, PythonInterpreter from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.platforms import Platform @@ -30,9 +30,9 @@ from pants.build_graph.files import Files from pants.build_graph.target import Target from pants.option.custom_types import UnsetBool -from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.python_setup import PythonSetup +from pants.python.python_repos import PythonRepos from pants.subsystem.subsystem import Subsystem from pants.util.collections import assert_single_element from pants.util.contextutil import temporary_file @@ -293,8 +293,8 @@ def add_resolved_requirements( ) else: self._log.debug( - f" Dumping distribution: .../{os.path.basename(dist.location)}" - ) + f" Dumping distribution: .../{os.path.basename(dist.location)}") + self.add_distribution(dist) locations.add(dist.location) # In addition to the top-level requirements, we add all the requirements matching the resolved @@ -435,15 +435,27 @@ def _prepare_inits(self) -> Set[str]: def set_emit_warnings(self, emit_warnings): self._builder.info.emit_warnings = emit_warnings + def _set_major_minor_interpreter_constraint_for_ipex( + self, + info: PexInfo, + identity: PythonIdentity, + ) -> PexInfo: + interpreter_name = identity.requirement.name + major, minor, _patch = identity.version + major_minor_only_constraint = f'{interpreter_name}=={major}.{minor}.*' + return ipex_launcher.modify_pex_info( + info, + interpreter_constraints=[str(major_minor_only_constraint)] + ) + def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]: """Replace the original builder with a new one, and just pull files from the old chroot.""" # Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is # (the exact same interpreter we used to resolve those requirements here). This is the only (?) # way to ensure that the ipex bootstrap uses the *exact* same interpreter version. - self._builder.info = ipex_launcher.modify_pex_info( + self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex( self._builder.info, - interpreter_constraints=[str(self._builder.interpreter.identity.requirement)], - ) + self._builder.interpreter.identity) orig_info = self._builder.info.copy() @@ -451,6 +463,9 @@ def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]: # Mutate the PexBuilder object which is manipulated by this subsystem. self._builder = PEXBuilder(interpreter=self._builder.interpreter) + self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex( + self._builder.info, + self._builder.interpreter.identity) return (orig_info, Path(orig_chroot.path())) diff --git a/src/python/pants/python/pex_build_util_test_integration.py b/src/python/pants/python/pex_build_util_test_integration.py new file mode 100644 index 00000000000..58e8d62916b --- /dev/null +++ b/src/python/pants/python/pex_build_util_test_integration.py @@ -0,0 +1,46 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import json +import os +import subprocess + +from pex.interpreter import PythonInterpreter + +from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest +from pants.util.collections import assert_single_element +from pants.util.contextutil import open_zip, temporary_dir + + +class PexBuildUtilIntegrationTest(PantsRunIntegrationTest): + + binary_target_address = "testprojects/src/python/python_targets:test" + + def test_multiplatform_python_binary(self): + cur_interpreter_id = PythonInterpreter.get().identity + interpreter_name = cur_interpreter_id.requirement.name + major, minor, patch = cur_interpreter_id.version + cur_interpreter_constraint = f"{interpreter_name}=={major}.{minor}.{patch}" + + imprecise_constraint = f"{interpreter_name}=={major}.{minor}.*" + + with temporary_dir() as tmp_dir: + self.do_command( + "--binary-py-generate-ipex", + "binary", + self.binary_target_address, + config={ + "GLOBAL": {"pants_distdir": tmp_dir}, + "python-setup": {"interpreter_constraints": [cur_interpreter_constraint],}, + }, + ) + + pex_path = os.path.join(tmp_dir, "test.ipex") + assert os.path.isfile(pex_path) + pex_execution_result = subprocess.run([pex_path], stdout=subprocess.PIPE, check=True) + assert pex_execution_result.stdout.decode() == "test!\n" + + with open_zip(pex_path) as zf: + info = json.loads(zf.read("PEX-INFO")) + constraint = assert_single_element(info["interpreter_constraints"]) + assert constraint == imprecise_constraint diff --git a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py index d0aab18cc29..60741dd0573 100644 --- a/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py +++ b/tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py @@ -12,9 +12,9 @@ from pants.backend.codegen.thrift.python.apache_thrift_py_gen import ApacheThriftPyGen from pants.backend.codegen.thrift.python.python_thrift_library import PythonThriftLibrary from pants.backend.python.interpreter_cache import PythonInterpreterCache +from pants.python.python_repos import PythonRepos from pants.backend.python.targets.python_library import PythonLibrary from pants.base.build_environment import get_buildroot -from pants.backend.python.subsystems.python_repos import PythonRepos from pants.testutil.subsystem.util import global_subsystem_instance from pants.testutil.task_test_base import TaskTestBase diff --git a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py index 02f46bb8719..d4342003a15 100644 --- a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py @@ -8,12 +8,12 @@ from pants.backend.native.register import rules as native_backend_rules from pants.backend.native.subsystems.libc_dev import LibcDev from pants.backend.native.subsystems.native_build_step import ToolchainVariant +from pants.python.python_repos import PythonRepos from pants.backend.python.tasks.build_local_python_distributions import ( BuildLocalPythonDistributions, ) from pants.backend.python.tasks.resolve_requirements import ResolveRequirements from pants.backend.python.tasks.select_interpreter import SelectInterpreter -from pants.backend.python.subsystems.python_repos import PythonRepos from pants.testutil.task_test_base import DeclarativeTaskTestMixin from pants.util.collections import assert_single_element from pants.util.enums import match