forked from pantsbuild/pants
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
cache python tools in ~/.cache/pants (pantsbuild#7236)
### Problem This runs for (on my laptop) about 16 seconds every time I do a `clean-all`: ``` 22:27:23 00:02 [native-compile] 22:27:23 00:02 [conan-prep] 22:27:23 00:02 [create-conan-pex] 22:27:39 00:18 [conan-fetch] ``` It doesn't seem like we need to be putting this tool in the task workdir as the python requirements list is pretty static. Conan in particular will be instantiated by invoking almost every goal, and it is a nontrivial piece of software to resolve each time. Also, we aren't mixing in interpreter identity to the generated pex filename, which is a bug that has so far gone undetected: see pantsbuild#7236 (comment). ### Solution - Take the `stable_json_sha1()` of the requirements of each python tool generated by `PythonToolPrepBase` to generate a fingerprinted pex filename. - Stick it in the pants cachedir so it doesn't get blown away by a clean-all. - Add an `--interpreter-constraints` option to pex tools (where previously the repo's `--python-setup-interpreter-constraints` were implicitly used). - Ensure the selected interpreter identity is mixed into the fingerprinted filename. - Add a test for the pex filename fingerprinting and that the pex can be successfully executed for python 2 and 3 constraints. ### Result A significant amount of time spent waiting after clean builds is removed, and pex tools can have their own interpreter constraints as necessary.
- Loading branch information
1 parent
4097052
commit 8069653
Showing
4 changed files
with
129 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
87 changes: 87 additions & 0 deletions
87
tests/python/pants_test/backend/python/tasks/test_python_tool.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import os | ||
import re | ||
|
||
from pants.backend.python.subsystems.python_tool_base import PythonToolBase | ||
from pants.backend.python.tasks.python_tool_prep_base import PythonToolInstance, PythonToolPrepBase | ||
from pants.task.task import Task | ||
from pants.util.contextutil import temporary_dir | ||
from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase | ||
|
||
|
||
class Tool(PythonToolBase): | ||
options_scope = 'test-tool' | ||
default_requirements = [ | ||
'pex==1.5.3', | ||
] | ||
default_entry_point = 'pex.bin.pex:main' | ||
|
||
|
||
class ToolInstance(PythonToolInstance): | ||
pass | ||
|
||
|
||
class ToolPrep(PythonToolPrepBase): | ||
options_scope = 'tool-prep-task' | ||
tool_subsystem_cls = Tool | ||
tool_instance_cls = ToolInstance | ||
|
||
|
||
class ToolTask(Task): | ||
options_scope = 'tool-task' | ||
|
||
@classmethod | ||
def prepare(cls, options, round_manager): | ||
super(ToolTask, cls).prepare(options, round_manager) | ||
round_manager.require_data(ToolPrep.tool_instance_cls) | ||
|
||
def execute(self): | ||
tool_for_pex = self.context.products.get_data(ToolPrep.tool_instance_cls) | ||
stdout, _, exit_code, _ = tool_for_pex.output(['--version']) | ||
assert re.match(r'.*\.pex 1.5.3', stdout) | ||
assert 0 == exit_code | ||
|
||
|
||
class PythonToolPrepTest(PythonTaskTestBase): | ||
|
||
@classmethod | ||
def task_type(cls): | ||
return ToolTask | ||
|
||
def _assert_tool_execution_for_python_version(self, use_py3=True): | ||
scope_string = '3' if use_py3 else '2' | ||
constraint_string = 'CPython>=3' if use_py3 else 'CPython<3' | ||
tool_prep_type = self.synthesize_task_subtype(ToolPrep, 'tp_scope_py{}'.format(scope_string)) | ||
with temporary_dir() as tmp_dir: | ||
context = self.context(for_task_types=[tool_prep_type], for_subsystems=[Tool], options={ | ||
'': { | ||
'pants_bootstrapdir': tmp_dir, | ||
}, | ||
'test-tool': { | ||
'interpreter_constraints': [constraint_string], | ||
}, | ||
}) | ||
tool_prep_task = tool_prep_type(context, os.path.join( | ||
self.pants_workdir, 'tp_py{}'.format(scope_string))) | ||
tool_prep_task.execute() | ||
# Check that the tool can be created and executed successfully. | ||
self.create_task(context).execute() | ||
pex_tool = context.products.get_data(ToolPrep.tool_instance_cls) | ||
# Check that our pex tool wrapper was constructed with the expected interpreter. | ||
self.assertTrue(pex_tool.interpreter.identity.matches(constraint_string)) | ||
return pex_tool | ||
|
||
def test_tool_execution(self): | ||
"""Test that python tools are fingerprinted by python interpreter.""" | ||
py3_pex_tool = self._assert_tool_execution_for_python_version(use_py3=True) | ||
py3_pex_tool_path = py3_pex_tool.pex.path() | ||
self.assertTrue(os.path.isdir(py3_pex_tool_path)) | ||
py2_pex_tool = self._assert_tool_execution_for_python_version(use_py3=False) | ||
py2_pex_tool_path = py2_pex_tool.pex.path() | ||
self.assertTrue(os.path.isdir(py2_pex_tool_path)) | ||
self.assertNotEqual(py3_pex_tool_path, py2_pex_tool_path) |