Skip to content

Commit

Permalink
ipex additional fixes for PEX-INFO interpreter constraints
Browse files Browse the repository at this point in the history
add an integration test!!
  • Loading branch information
cosmicexplorer committed Mar 31, 2020
1 parent 1776dea commit c031e05
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 10 deletions.
30 changes: 30 additions & 0 deletions src/python/pants/python/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,33 @@ python_library(
'src/python/pants/util:memo',
]
)


python_tests(
name='tests',
sources=['*_test.py', '!*_integration.py'],
dependencies=[
'3rdparty/python:pex',
':python',
'src/python/pants/backend/python/targets',
'src/python/pants/build_graph',
'src/python/pants/util:contextutil',
'src/python/pants/testutil/subsystem',
'src/python/pants/testutil:pexrc_util',
'src/python/pants/testutil:test_base',
],
tags = {'partially_type_checked'},
)


python_tests(
name='integration',
sources=['*_integration.py'],
dependencies=[
'3rdparty/python:pex',
'src/python/pants/util:contextutil',
'src/python/pants/testutil:int-test',
'testprojects/src/python:python_targets_directory',
],
tags = {'integration', 'partially_type_checked'},
)
31 changes: 23 additions & 8 deletions src/python/pants/python/pex_build_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
from pathlib import Path
from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple

from pex.fetcher import PyPIFetcher, Fetcher
from pex.interpreter import PythonInterpreter
from pex.fetcher import Fetcher, PyPIFetcher
from pex.interpreter import PythonIdentity, PythonInterpreter
from pex.pex_builder import PEXBuilder
from pex.pex_info import PexInfo
from pex.platforms import Platform
Expand All @@ -30,9 +30,9 @@
from pants.build_graph.files import Files
from pants.build_graph.target import Target
from pants.option.custom_types import UnsetBool
from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.backend.python.python_requirement import PythonRequirement
from pants.backend.python.subsystems.python_setup import PythonSetup
from pants.python.python_repos import PythonRepos
from pants.subsystem.subsystem import Subsystem
from pants.util.collections import assert_single_element
from pants.util.contextutil import temporary_file
Expand Down Expand Up @@ -293,8 +293,8 @@ def add_resolved_requirements(
)
else:
self._log.debug(
f" Dumping distribution: .../{os.path.basename(dist.location)}"
)
f" Dumping distribution: .../{os.path.basename(dist.location)}")

self.add_distribution(dist)
locations.add(dist.location)
# In addition to the top-level requirements, we add all the requirements matching the resolved
Expand Down Expand Up @@ -435,22 +435,37 @@ def _prepare_inits(self) -> Set[str]:
def set_emit_warnings(self, emit_warnings):
self._builder.info.emit_warnings = emit_warnings

def _set_major_minor_interpreter_constraint_for_ipex(
self,
info: PexInfo,
identity: PythonIdentity,
) -> PexInfo:
interpreter_name = identity.requirement.name
major, minor, _patch = identity.version
major_minor_only_constraint = f'{interpreter_name}=={major}.{minor}.*'
return ipex_launcher.modify_pex_info(
info,
interpreter_constraints=[str(major_minor_only_constraint)]
)

def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]:
"""Replace the original builder with a new one, and just pull files from the old chroot."""
# Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is
# (the exact same interpreter we used to resolve those requirements here). This is the only (?)
# way to ensure that the ipex bootstrap uses the *exact* same interpreter version.
self._builder.info = ipex_launcher.modify_pex_info(
self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
self._builder.info,
interpreter_constraints=[str(self._builder.interpreter.identity.requirement)],
)
self._builder.interpreter.identity)

orig_info = self._builder.info.copy()

orig_chroot = self._builder.chroot()

# Mutate the PexBuilder object which is manipulated by this subsystem.
self._builder = PEXBuilder(interpreter=self._builder.interpreter)
self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
self._builder.info,
self._builder.interpreter.identity)

return (orig_info, Path(orig_chroot.path()))

Expand Down
46 changes: 46 additions & 0 deletions src/python/pants/python/pex_build_util_test_integration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import json
import os
import subprocess

from pex.interpreter import PythonInterpreter

from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest
from pants.util.collections import assert_single_element
from pants.util.contextutil import open_zip, temporary_dir


class PexBuildUtilIntegrationTest(PantsRunIntegrationTest):

binary_target_address = "testprojects/src/python/python_targets:test"

def test_multiplatform_python_binary(self):
cur_interpreter_id = PythonInterpreter.get().identity
interpreter_name = cur_interpreter_id.requirement.name
major, minor, patch = cur_interpreter_id.version
cur_interpreter_constraint = f"{interpreter_name}=={major}.{minor}.{patch}"

imprecise_constraint = f"{interpreter_name}=={major}.{minor}.*"

with temporary_dir() as tmp_dir:
self.do_command(
"--binary-py-generate-ipex",
"binary",
self.binary_target_address,
config={
"GLOBAL": {"pants_distdir": tmp_dir},
"python-setup": {"interpreter_constraints": [cur_interpreter_constraint],},
},
)

pex_path = os.path.join(tmp_dir, "test.ipex")
assert os.path.isfile(pex_path)
pex_execution_result = subprocess.run([pex_path], stdout=subprocess.PIPE, check=True)
assert pex_execution_result.stdout.decode() == "test!\n"

with open_zip(pex_path) as zf:
info = json.loads(zf.read("PEX-INFO"))
constraint = assert_single_element(info["interpreter_constraints"])
assert constraint == imprecise_constraint
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
from pants.backend.codegen.thrift.python.apache_thrift_py_gen import ApacheThriftPyGen
from pants.backend.codegen.thrift.python.python_thrift_library import PythonThriftLibrary
from pants.backend.python.interpreter_cache import PythonInterpreterCache
from pants.python.python_repos import PythonRepos
from pants.backend.python.targets.python_library import PythonLibrary
from pants.base.build_environment import get_buildroot
from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.testutil.subsystem.util import global_subsystem_instance
from pants.testutil.task_test_base import TaskTestBase

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@
from pants.backend.native.register import rules as native_backend_rules
from pants.backend.native.subsystems.libc_dev import LibcDev
from pants.backend.native.subsystems.native_build_step import ToolchainVariant
from pants.python.python_repos import PythonRepos
from pants.backend.python.tasks.build_local_python_distributions import (
BuildLocalPythonDistributions,
)
from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
from pants.backend.python.tasks.select_interpreter import SelectInterpreter
from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.testutil.task_test_base import DeclarativeTaskTestMixin
from pants.util.collections import assert_single_element
from pants.util.enums import match
Expand Down

0 comments on commit c031e05

Please sign in to comment.