Skip to content

Commit

Permalink
Add RuleRunner.get_target() (#10714)
Browse files Browse the repository at this point in the history
We often request a target in tests for some address. This is common enough to be worth factoring up.

We also port `run_setup_py_test.py` to use Pytest style, and to use this new util. In addition, we make some small improvements to `run_setup_py.py`. This is prework for #10633.
  • Loading branch information
Eric-Arellano authored Sep 1, 2020
1 parent b543adb commit 6b79447
Show file tree
Hide file tree
Showing 10 changed files with 653 additions and 640 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
from pants.engine.addresses import Address
from pants.engine.fs import DigestContents
from pants.engine.rules import QueryRule
from pants.engine.target import WrappedTarget
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.testutil.option_util import create_options_bootstrapper
from pants.testutil.rule_runner import RuleRunner
Expand All @@ -40,7 +39,7 @@ def create_python_awslambda(rule_runner: RuleRunner, addr: str) -> Tuple[str, by
"--source-root-patterns=src/python",
]
)
target = rule_runner.request_product(WrappedTarget, [Address.parse(addr), bootstrapper]).target
target = rule_runner.get_target(Address.parse(addr), bootstrapper)
created_awslambda = rule_runner.request_product(
CreatedAWSLambda, [PythonAwsLambdaFieldSet.create(target), bootstrapper]
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from pants.core.util_rules import stripped_source_files
from pants.engine.addresses import Address
from pants.engine.rules import QueryRule, SubsystemRule
from pants.engine.target import InferredDependencies, WrappedTarget
from pants.engine.target import InferredDependencies
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.testutil.option_util import create_options_bootstrapper
from pants.testutil.rule_runner import RuleRunner
Expand Down Expand Up @@ -91,7 +91,7 @@ def run_dep_inference(
if enable_string_imports:
args.append("--python-infer-string-imports")
options_bootstrapper = create_options_bootstrapper(args=args)
target = rule_runner.request_product(WrappedTarget, [address, options_bootstrapper]).target
target = rule_runner.get_target(address, options_bootstrapper)
return rule_runner.request_product(
InferredDependencies,
[InferPythonDependencies(target[PythonSources]), options_bootstrapper],
Expand Down Expand Up @@ -152,7 +152,7 @@ def test_infer_python_inits() -> None:
rule_runner.add_to_build_file("src/python/root/mid/leaf", "python_library()")

def run_dep_inference(address: Address) -> InferredDependencies:
target = rule_runner.request_product(WrappedTarget, [address, options_bootstrapper]).target
target = rule_runner.get_target(address, options_bootstrapper)
return rule_runner.request_product(
InferredDependencies,
[InferInitDependencies(target[PythonSources]), options_bootstrapper],
Expand Down Expand Up @@ -195,7 +195,7 @@ def test_infer_python_conftests() -> None:
rule_runner.add_to_build_file("src/python/root/mid/leaf", "python_tests()")

def run_dep_inference(address: Address) -> InferredDependencies:
target = rule_runner.request_product(WrappedTarget, [address, options_bootstrapper]).target
target = rule_runner.get_target(address, options_bootstrapper)
return rule_runner.request_product(
InferredDependencies,
[InferConftestDependencies(target[PythonSources]), options_bootstrapper],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from pants.engine.addresses import Address
from pants.engine.fs import FileContent
from pants.engine.rules import QueryRule
from pants.engine.target import Target, WrappedTarget
from pants.engine.target import Target
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.testutil.option_util import create_options_bootstrapper
from pants.testutil.python_interpreter_selection import skip_unless_python27_and_python3_present
Expand Down Expand Up @@ -71,13 +71,9 @@ def make_target(
"""
),
)
return rule_runner.request_product(
WrappedTarget,
[
Address(package, target_name=name),
create_options_bootstrapper(args=GLOBAL_ARGS),
],
).target
return rule_runner.get_target(
Address(package, target_name=name), create_options_bootstrapper(args=GLOBAL_ARGS)
)


def run_pylint(
Expand Down
11 changes: 3 additions & 8 deletions src/python/pants/backend/python/pants_requirement_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from pants.base.build_environment import pants_version
from pants.engine.addresses import Address
from pants.engine.internals.scheduler import ExecutionError
from pants.engine.target import WrappedTarget
from pants.testutil.option_util import create_options_bootstrapper
from pants.testutil.rule_runner import RuleRunner
from pants.util.frozendict import FrozenDict
Expand All @@ -36,13 +35,9 @@ def assert_pants_requirement(
expected_module: str = "pants",
) -> None:
rule_runner.add_to_build_file("3rdparty/python", f"{build_file_entry}\n")
target = rule_runner.request_product(
WrappedTarget,
[
Address("3rdparty/python", target_name=expected_target_name),
create_options_bootstrapper(),
],
).target
target = rule_runner.get_target(
Address("3rdparty/python", target_name=expected_target_name), create_options_bootstrapper()
)
assert isinstance(target, PythonRequirementLibrary)
assert target[PythonRequirementsField].value == (
Requirement.parse(f"{expected_dist}=={pants_version()}"),
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/backend/python/python_artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def requirement(self):
return f"{self._name}=={self._version}"

@property
def setup_py_keywords(self):
def kwargs(self):
return self._kw

@property
Expand Down
83 changes: 43 additions & 40 deletions src/python/pants/backend/python/rules/run_setup_py.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,16 @@
PythonRequirementsField,
PythonSources,
)
from pants.base.specs import AddressLiteralSpec, AddressSpecs, AscendantAddresses
from pants.base.specs import (
AddressLiteralSpec,
AddressSpecs,
AscendantAddresses,
FilesystemLiteralSpec,
)
from pants.core.target_types import FilesSources, ResourcesSources
from pants.core.util_rules.distdir import DistDir
from pants.engine.addresses import Address, Addresses, AddressInput
from pants.engine.collection import Collection, DeduplicatedCollection
from pants.engine.console import Console
from pants.engine.fs import (
AddPrefix,
CreateDigest,
Expand Down Expand Up @@ -220,18 +224,19 @@ def register_options(cls, register):
type=list,
member_type=shell_str,
passthrough=True,
help="Arguments to pass directly to setup.py, e.g. "
'`--setup-py-args="bdist_wheel --python-tag py36.py37"`. If unspecified, Pants will '
"dump the setup.py chroot.",
help=(
"Arguments to pass directly to setup.py, e.g. `--setup-py-args='bdist_wheel "
"--python-tag py36.py37'`. If unspecified, Pants will dump the setup.py chroot."
),
)
register(
"--transitive",
type=bool,
default=False,
help="If specified, will run the setup.py command recursively on all exported targets that "
"the specified targets depend on, in dependency order. This is useful, e.g., when "
"the command publishes dists, to ensure that any dependencies of a dist are published "
"before it.",
help=(
"If specified, will run the setup.py command recursively on all exported targets "
"that the specified targets depend on, in dependency order."
),
)

@property
Expand Down Expand Up @@ -269,7 +274,6 @@ def validate_args(args: Tuple[str, ...]):
async def run_setup_pys(
targets_with_origins: TargetsWithOrigins,
setup_py_subsystem: SetupPySubsystem,
console: Console,
python_setup: PythonSetup,
distdir: DistDir,
workspace: Workspace,
Expand All @@ -288,7 +292,7 @@ async def run_setup_pys(
tgt = target_with_origin.target
if tgt.has_field(PythonProvidesField):
exported_targets.append(ExportedTarget(tgt))
elif isinstance(target_with_origin.origin, AddressLiteralSpec):
elif isinstance(target_with_origin.origin, (AddressLiteralSpec, FilesystemLiteralSpec)):
explicit_nonexported_targets.append(tgt)
if explicit_nonexported_targets:
raise TargetNotExported(
Expand Down Expand Up @@ -331,15 +335,15 @@ async def run_setup_pys(

for exported_target, setup_py_result in zip(exported_targets, setup_py_results):
addr = exported_target.target.address.spec
console.print_stderr(f"Writing dist for {addr} under {distdir.relpath}/.")
logger.info(f"Writing dist for {addr} under {distdir.relpath}/.")
workspace.write_digest(setup_py_result.output, path_prefix=str(distdir.relpath))
else:
# Just dump the chroot.
for exported_target, chroot in zip(exported_targets, chroots):
addr = exported_target.target.address.spec
provides = exported_target.provides
setup_py_dir = distdir.relpath / f"{provides.name}-{provides.version}"
console.print_stderr(f"Writing setup.py chroot for {addr} to {setup_py_dir}")
logger.info(f"Writing setup.py chroot for {addr} to {setup_py_dir}")
workspace.write_digest(chroot.digest, path_prefix=str(setup_py_dir))

return SetupPy(0)
Expand Down Expand Up @@ -390,24 +394,27 @@ async def run_setup_py(
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:
exported_target = request.exported_target

owned_deps = await Get(OwnedDependencies, DependencyOwner(exported_target))
transitive_targets = await Get(TransitiveTargets, Addresses([exported_target.target.address]))
owned_deps, transitive_targets = await MultiGet(
Get(OwnedDependencies, DependencyOwner(exported_target)),
Get(TransitiveTargets, Addresses([exported_target.target.address])),
)

# files() targets aren't owned by a single exported target - they aren't code, so
# we allow them to be in multiple dists. This is helpful for, e.g., embedding
# a standard license file in a dist.
files_targets = (tgt for tgt in transitive_targets.closure if tgt.has_field(FilesSources))
targets = Targets(itertools.chain((od.target for od in owned_deps), files_targets))
sources = await Get(SetupPySources, SetupPySourcesRequest(targets, py2=request.py2))
requirements = await Get(ExportedTargetRequirements, DependencyOwner(exported_target))

# Nest the sources under the src/ prefix.
src_digest = await Get(Digest, AddPrefix(sources.digest, CHROOT_SOURCE_ROOT))
sources, requirements = await MultiGet(
Get(SetupPySources, SetupPySourcesRequest(targets, py2=request.py2)),
Get(ExportedTargetRequirements, DependencyOwner(exported_target)),
)

target = exported_target.target
provides = exported_target.provides

# Generate the kwargs to the setup() call.
setup_kwargs = provides.setup_py_keywords.copy()
setup_kwargs = provides.kwargs.copy()
setup_kwargs.update(
{
"package_dir": {"": CHROOT_SOURCE_ROOT},
Expand Down Expand Up @@ -443,16 +450,14 @@ async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:
target_address_spec=target.address.spec,
setup_kwargs_str=distutils_repr(setup_kwargs),
).encode()
extra_files_digest = await Get(
Digest,
CreateDigest(
[
FileContent("setup.py", setup_py_content),
FileContent(
"MANIFEST.in", "include *.py".encode()
), # Make sure setup.py is included.
]
),
files_to_create = [
FileContent("setup.py", setup_py_content),
FileContent("MANIFEST.in", "include *.py".encode()),
]
extra_files_digest, src_digest = await MultiGet(
Get(Digest, CreateDigest(files_to_create)),
# Nest the sources under the src/ prefix.
Get(Digest, AddPrefix(sources.digest, CHROOT_SOURCE_ROOT)),
)

chroot_digest = await Get(Digest, MergeDigests((src_digest, extra_files_digest)))
Expand All @@ -461,17 +466,15 @@ async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:

@rule
async def get_sources(request: SetupPySourcesRequest) -> SetupPySources:
python_sources = await Get(
StrippedPythonSourceFiles,
PythonSourceFilesRequest(
targets=request.targets, include_resources=False, include_files=False
),
python_sources_request = PythonSourceFilesRequest(
targets=request.targets, include_resources=False, include_files=False
)
all_sources = await Get(
StrippedPythonSourceFiles,
PythonSourceFilesRequest(
targets=request.targets, include_resources=True, include_files=True
),
all_sources_request = PythonSourceFilesRequest(
targets=request.targets, include_resources=True, include_files=True
)
python_sources, all_sources = await MultiGet(
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, python_sources_request),
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, all_sources_request),
)

python_files = set(python_sources.stripped_source_files.snapshot.files)
Expand Down
Loading

0 comments on commit 6b79447

Please sign in to comment.