V2 run

src/python/pants/backend/python/register.py

@@ -5,7 +5,14 @@
 from pants.backend.python.python_artifact import PythonArtifact
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.python_requirements import PythonRequirements
-from pants.backend.python.rules import download_pex_bin, inject_init, pex, python_test_runner
+from pants.backend.python.rules import (
+  download_pex_bin,
+  inject_init,
+  interpreter_constraints,
+  pex,
+  python_run_binary,
+  python_test_runner,
+)
 from pants.backend.python.subsystems.python_native_code import PythonNativeCode
 from pants.backend.python.subsystems.python_native_code import rules as python_native_code_rules
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEnvironment
@@ -93,7 +100,9 @@ def rules():
   return (
     download_pex_bin.rules() +
     inject_init.rules() +
+    interpreter_constraints.rules() +
     python_test_runner.rules() +
+    python_run_binary.rules() +
     python_native_code_rules() +
     pex.rules() +
     subprocess_environment_rules()

src/python/pants/backend/python/rules/interpreter_constraints.py

@@ -0,0 +1,42 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+from typing import FrozenSet, List, Tuple
+
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.engine.legacy.structs import PythonTargetAdaptor
+from pants.engine.rules import RootRule, rule
+
+
+@dataclass(frozen=True)
+class BuildConstraintsForAdaptors:
+  adaptors: Tuple[PythonTargetAdaptor]
+
+
+@dataclass(frozen=True)
+class PexInterpreterContraints:
+  constraint_set: FrozenSet[str] = frozenset()
+
+  def generate_pex_arg_list(self) -> List[str]:
+    args = []
+    for constraint in self.constraint_set:
+      args.extend(["--interpreter-constraint", constraint])
+    return args
+
+
+@rule
+def handle_constraints(build_constraints_for_adaptors: BuildConstraintsForAdaptors, python_setup: PythonSetup) -> PexInterpreterContraints:
+  interpreter_constraints = frozenset(
+    [constraint
+    for target_adaptor in build_constraints_for_adaptors.adaptors
+    for constraint in python_setup.compatibility_or_constraints(
+      getattr(target_adaptor, 'compatibility', None)
+    )]
+  )
+
+  yield PexInterpreterContraints(constraint_set=interpreter_constraints)
+
+
+def rules():
+  return [handle_constraints, RootRule(BuildConstraintsForAdaptors)]

src/python/pants/backend/python/rules/pex.py

@@ -6,6 +6,7 @@
 
 from pants.backend.python.rules.download_pex_bin import DownloadedPexBin
 from pants.backend.python.rules.hermetic_pex import HermeticPex
+from pants.backend.python.rules.interpreter_constraints import PexInterpreterContraints
 from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
@@ -15,18 +16,23 @@
   DirectoriesToMerge,
   DirectoryWithPrefixToAdd,
 )
-from pants.engine.isolated_process import ExecuteProcessResult, MultiPlatformExecuteProcessRequest
+from pants.engine.isolated_process import (
+  ExecuteProcessRequest,
+  ExecuteProcessResult,
+  MultiPlatformExecuteProcessRequest,
+)
 from pants.engine.platform import Platform, PlatformConstraint
-from pants.engine.rules import optionable_rule, rule
+from pants.engine.rules import RootRule, optionable_rule, rule
 from pants.engine.selectors import Get
+from pants.util.strutil import create_path_env_var
 
 
 @dataclass(frozen=True)
 class CreatePex:
   """Represents a generic request to create a PEX from its inputs."""
   output_filename: str
   requirements: Tuple[str] = ()
-  interpreter_constraints: Tuple[str] = ()
+  interpreter_constraints: PexInterpreterContraints = PexInterpreterContraints()
   entry_point: Optional[str] = None
   input_files_digest: Optional[Digest] = None
 
@@ -35,6 +41,7 @@ class CreatePex:
 class Pex(HermeticPex):
   """Wrapper for a digest containing a pex file created with some filename."""
   directory_digest: Digest
+  output_filename: str
 
 
 # TODO: This is non-hermetic because the requirements will be resolved on the fly by
@@ -51,9 +58,7 @@ def create_pex(
   """Returns a PEX with the given requirements, optional entry point, and optional
   interpreter constraints."""
 
-  interpreter_constraint_args = []
-  for constraint in request.interpreter_constraints:
-    interpreter_constraint_args.extend(["--interpreter-constraint", constraint])
+  interpreter_constraint_args = request.interpreter_constraints.generate_pex_arg_list()
 
   argv = ["--output-file", request.output_filename]
   if request.entry_point is not None:
@@ -97,11 +102,43 @@ def create_pex(
     MultiPlatformExecuteProcessRequest,
     execute_process_request
   )
-  yield Pex(directory_digest=result.output_directory_digest)
+  yield Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename)
+
+
+@dataclass(frozen=True)
+class RunnablePex:
+  pex: CreatePex
+
+
+# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
+# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
+# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
+# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
+# necessarily the interpreter that PEX will use to execute the generated .pex file.
+@rule
+def pex_execute_request(
+  runnable_pex: RunnablePex, #TODO if this rule is a Pex the rule graph blows up wtf?
+  subprocess_encoding_environment: SubprocessEncodingEnvironment,
+  python_setup: PythonSetup
+  ) -> ExecuteProcessRequest:
+
+  entry_point = runnable_pex.pex.entry_point
+  pex = yield Get(Pex, CreatePex, runnable_pex.pex)
+  interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths)
+  pex_exe_env = { 'PATH': interpreter_search_paths, **subprocess_encoding_environment.invocation_environment_dict }
+  request = ExecuteProcessRequest(
+    argv=("python", f'./{pex.output_filename}'),
+    env=pex_exe_env,
+    input_files=pex.directory_digest,
+    description=f'Run {entry_point} as PEX'
+  )
+  yield request
 
 
 def rules():
   return [
     create_pex,
+    pex_execute_request,
     optionable_rule(PythonSetup),
+    RootRule(RunnablePex),
   ]

src/python/pants/backend/python/rules/python_run_binary.py

@@ -0,0 +1,96 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.rules.inject_init import InjectedInitDigest
+from pants.backend.python.rules.interpreter_constraints import (
+  BuildConstraintsForAdaptors,
+  PexInterpreterContraints,
+)
+from pants.backend.python.rules.pex import CreatePex, RunnablePex
+from pants.backend.python.subsystems.python_setup import PythonSetup
+from pants.backend.python.targets.python_binary import PythonBinary
+from pants.engine.fs import Digest, DirectoriesToMerge
+from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult
+from pants.engine.legacy.graph import BuildFileAddresses, HydratedTarget, TransitiveHydratedTargets
+from pants.engine.legacy.structs import PythonBinaryAdaptor
+from pants.engine.rules import UnionRule, rule
+from pants.engine.selectors import Get
+from pants.rules.core.run import RunResult, RunTarget
+from pants.rules.core.strip_source_root import SourceRootStrippedSources
+
+
+@rule
+def run_python_binary(python_binary_target: PythonBinaryAdaptor,
+  python_setup: PythonSetup) -> RunResult:
+
+  transitive_hydrated_targets = yield Get(
+    TransitiveHydratedTargets, BuildFileAddresses((python_binary_target.address,))
+  )
+  all_targets = transitive_hydrated_targets.closure
+  all_target_adaptors = [t.adaptor for t in all_targets]
+
+
+  interpreter_constraints = yield Get(PexInterpreterContraints,
+      BuildConstraintsForAdaptors(adaptors=tuple(all_target_adaptors)))
+
+  source_root_stripped_sources = yield [
+    Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
+    for target_adaptor in all_targets
+  ]
+
+  #TODO This way of calculating the entry point works but is a bit hackish.
+  entry_point = None
+  if hasattr(python_binary_target, 'entry_point'):
+    entry_point = python_binary_target.entry_point
+  else:
+    sources_snapshot = python_binary_target.sources.snapshot
+    if len(sources_snapshot.files) == 1:
+      target = transitive_hydrated_targets.roots[0]
+      output = yield Get(SourceRootStrippedSources, HydratedTarget, target)
+      root_filename = output.snapshot.files[0]
+      entry_point = PythonBinary.translate_source_path_to_py_module_specifier(root_filename)
+
+  stripped_sources_digests = [stripped_sources.snapshot.directory_digest for stripped_sources in source_root_stripped_sources]
+  sources_digest = yield Get(Digest, DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
+  inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
+  all_input_digests = [sources_digest, inits_digest.directory_digest]
+  merged_input_files = yield Get(Digest, DirectoriesToMerge, DirectoriesToMerge(directories=tuple(all_input_digests)))
+
+  #TODO This chunk of code should be made into an @rule and used both here and in
+  # python_test_runner.py.
+  # Produce a pex containing pytest and all transitive 3rdparty requirements.
+  output_thirdparty_requirements_pex_filename = '3rdparty-requirements.pex'
+  all_target_requirements = []
+  for maybe_python_req_lib in all_target_adaptors:
+    # This is a python_requirement()-like target.
+    if hasattr(maybe_python_req_lib, 'requirement'):
+      all_target_requirements.append(str(maybe_python_req_lib.requirement))
+    # This is a python_requirement_library()-like target.
+    if hasattr(maybe_python_req_lib, 'requirements'):
+      for py_req in maybe_python_req_lib.requirements:
+        all_target_requirements.append(str(py_req.requirement))
+
+  all_requirements = all_target_requirements
+  create_requirements_pex = CreatePex(
+    output_filename=output_thirdparty_requirements_pex_filename,
+    requirements=tuple(sorted(all_requirements)),
+    interpreter_constraints=interpreter_constraints,
+    entry_point=entry_point,
+    input_files_digest=merged_input_files,
+  )
+
+  exec_request = yield Get(ExecuteProcessRequest, RunnablePex(pex=create_requirements_pex))
+  result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, exec_request)
+
+  yield RunResult(
+    status=result.exit_code,
+    stdout=result.stdout.decode(),
+    stderr=result.stderr.decode(),
+  )
+
+
+def rules():
+  return [
+    run_python_binary,
+    UnionRule(RunTarget, PythonBinaryAdaptor),
+  ]

src/python/pants/backend/python/rules/python_test_runner.py

@@ -2,6 +2,10 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.rules.inject_init import InjectedInitDigest
+from pants.backend.python.rules.interpreter_constraints import (
+  BuildConstraintsForAdaptors,
+  PexInterpreterContraints,
+)
 from pants.backend.python.rules.pex import CreatePex, Pex
 from pants.backend.python.subsystems.pytest import PyTest
 from pants.backend.python.subsystems.python_setup import PythonSetup
@@ -33,13 +37,8 @@ def run_python_test(
   )
   all_targets = transitive_hydrated_targets.closure
 
-  interpreter_constraints = {
-    constraint
-    for target_adaptor in all_targets
-    for constraint in python_setup.compatibility_or_constraints(
-      getattr(target_adaptor, 'compatibility', None)
-    )
-  }
+  interpreter_constraints = yield Get(PexInterpreterContraints,
+      BuildConstraintsForAdaptors(adaptors=tuple(all_targets)))
 
   # Produce a pex containing pytest and all transitive 3rdparty requirements.
   output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
@@ -58,7 +57,7 @@ def run_python_test(
     Pex, CreatePex(
       output_filename=output_pytest_requirements_pex_filename,
       requirements=tuple(sorted(all_requirements)),
-      interpreter_constraints=tuple(sorted(interpreter_constraints)),
+      interpreter_constraints=interpreter_constraints,
       entry_point="pytest:main",
     )
   )

src/python/pants/backend/python/targets/python_binary.py

@@ -116,7 +116,7 @@ def __init__(self,
     if sources and sources.files and entry_point:
       entry_point_module = entry_point.split(':', 1)[0]
       entry_source = list(self.sources_relative_to_source_root())[0]
-      source_entry_point = self._translate_to_entry_point(entry_source)
+      source_entry_point = self.translate_source_path_to_py_module_specifier(entry_source)
       if entry_point_module != source_entry_point:
         raise TargetDefinitionException(self,
             'Specified both source and entry_point but they do not agree: {} vs {}'.format(
@@ -136,7 +136,8 @@ def repositories(self):
   def indices(self):
     return self.payload.indices
 
-  def _translate_to_entry_point(self, source):
+  @classmethod
+  def translate_source_path_to_py_module_specifier(self, source: str) -> str:
     source_base, _ = os.path.splitext(source)
     return source_base.replace(os.path.sep, '.')
 
@@ -147,7 +148,7 @@ def entry_point(self):
     elif self.payload.sources.source_paths:
       assert len(self.payload.sources.source_paths) == 1
       entry_source = list(self.sources_relative_to_source_root())[0]
-      return self._translate_to_entry_point(entry_source)
+      return self.translate_source_path_to_py_module_specifier(entry_source)
     else:
       return None
 

src/python/pants/rules/core/register.py

@@ -1,14 +1,15 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.rules.core import filedeps, list_roots, list_targets, strip_source_root, test
+from pants.rules.core import filedeps, list_roots, list_targets, run, strip_source_root, test
 
 
 def rules():
   return [
     *list_roots.rules(),
     *list_targets.rules(),
     *filedeps.rules(),
+    *run.rules(),
     *strip_source_root.rules(),
     *test.rules()
   ]