diff --git a/src/python/pants/backend/awslambda/python/rules.py b/src/python/pants/backend/awslambda/python/rules.py index 54589a4aa05..0798b2b5d66 100644 --- a/src/python/pants/backend/awslambda/python/rules.py +++ b/src/python/pants/backend/awslambda/python/rules.py @@ -10,6 +10,9 @@ PythonAWSLambda, PythonAwsLambdaHandlerField, PythonAwsLambdaIncludeRequirements, + PythonAwsLambdaIncludeSources, + PythonAWSLambdaLayer, + PythonAwsLambdaLayerDependenciesField, PythonAwsLambdaRuntime, ) from pants.backend.python.subsystems.lambdex import Lambdex, LambdexLayout @@ -22,17 +25,16 @@ from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet from pants.core.util_rules.environments import EnvironmentField from pants.engine.rules import Get, collect_rules, rule +from pants.engine.target import InvalidTargetException from pants.engine.unions import UnionRule from pants.util.logging import LogLevel +from pants.util.strutil import softwrap logger = logging.getLogger(__name__) @dataclass(frozen=True) -class PythonAwsLambdaFieldSet(PackageFieldSet): - required_fields = (PythonAwsLambdaHandlerField,) - - handler: PythonAwsLambdaHandlerField +class _BaseFieldSet(PackageFieldSet): include_requirements: PythonAwsLambdaIncludeRequirements runtime: PythonAwsLambdaRuntime complete_platforms: PythonFaaSCompletePlatforms @@ -40,6 +42,21 @@ class PythonAwsLambdaFieldSet(PackageFieldSet): environment: EnvironmentField +@dataclass(frozen=True) +class PythonAwsLambdaFieldSet(_BaseFieldSet): + required_fields = (PythonAwsLambdaHandlerField,) + + handler: PythonAwsLambdaHandlerField + + +@dataclass(frozen=True) +class PythonAwsLambdaLayerFieldSet(_BaseFieldSet): + required_fields = (PythonAwsLambdaLayerDependenciesField,) + + dependencies: PythonAwsLambdaLayerDependenciesField + include_sources: PythonAwsLambdaIncludeSources + + @rule(desc="Create Python AWS Lambda", level=LogLevel.DEBUG) async def package_python_awslambda( field_set: PythonAwsLambdaFieldSet, @@ -74,11 +91,56 @@ async def package_python_awslambda( handler=field_set.handler, output_path=field_set.output_path, include_requirements=field_set.include_requirements.value, + include_sources=True, reexported_handler_module=PythonAwsLambdaHandlerField.reexported_handler_module, ), ) +@rule(desc="Create Python AWS Lambda Layer", level=LogLevel.DEBUG) +async def package_python_aws_lambda_layer( + field_set: PythonAwsLambdaLayerFieldSet, + lambdex: Lambdex, +) -> BuiltPackage: + if lambdex.layout is LambdexLayout.LAMBDEX: + raise InvalidTargetException( + softwrap( + f""" + the `{PythonAWSLambdaLayer.alias}` target {field_set.address} cannot be used with + the old Lambdex layout (`[lambdex].layout = \"{LambdexLayout.LAMBDEX.value}\"` in + `pants.toml`), set that to `{LambdexLayout.ZIP.value}` or remove this target + """ + ) + ) + + return await Get( + BuiltPackage, + BuildPythonFaaSRequest( + address=field_set.address, + target_name=PythonAWSLambdaLayer.alias, + complete_platforms=field_set.complete_platforms, + runtime=field_set.runtime, + output_path=field_set.output_path, + include_requirements=field_set.include_requirements.value, + include_sources=field_set.include_sources.value, + # See + # https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path + # + # Runtime | Path + # ... + # Python | `python` + # | `python/lib/python3.10/site-packages` + # ... + # + # The one independent on the runtime-version is more convenient: + prefix_in_artifact="python", + # a layer doesn't have a handler, just pulls in things via `dependencies` + handler=None, + reexported_handler_module=None, + ), + ) + + def rules(): return [ *collect_rules(), diff --git a/src/python/pants/backend/awslambda/python/rules_test.py b/src/python/pants/backend/awslambda/python/rules_test.py index 8de3a9f88e6..4a8a87a3135 100644 --- a/src/python/pants/backend/awslambda/python/rules_test.py +++ b/src/python/pants/backend/awslambda/python/rules_test.py @@ -12,9 +12,12 @@ import pytest -from pants.backend.awslambda.python.rules import PythonAwsLambdaFieldSet +from pants.backend.awslambda.python.rules import ( + PythonAwsLambdaFieldSet, + PythonAwsLambdaLayerFieldSet, +) from pants.backend.awslambda.python.rules import rules as awslambda_python_rules -from pants.backend.awslambda.python.target_types import PythonAWSLambda +from pants.backend.awslambda.python.target_types import PythonAWSLambda, PythonAWSLambdaLayer from pants.backend.awslambda.python.target_types import rules as target_rules from pants.backend.python.goals import package_pex_binary from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet @@ -37,6 +40,8 @@ from pants.core.target_types import rules as core_target_types_rules from pants.engine.addresses import Address from pants.engine.fs import DigestContents +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.target import FieldSet from pants.testutil.python_interpreter_selection import all_major_minor_python_versions from pants.testutil.python_rule_runner import PythonRuleRunner from pants.testutil.rule_runner import QueryRule @@ -54,12 +59,14 @@ def rule_runner() -> PythonRuleRunner: *target_rules(), *package.rules(), QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet,)), + QueryRule(BuiltPackage, (PythonAwsLambdaLayerFieldSet,)), ], target_types=[ FileTarget, FilesGeneratorTarget, PexBinary, PythonAWSLambda, + PythonAWSLambdaLayer, PythonRequirementTarget, PythonRequirementTarget, PythonSourcesGeneratorTarget, @@ -77,13 +84,18 @@ def create_python_awslambda( *, expected_extra_log_lines: tuple[str, ...], extra_args: list[str] | None = None, + layer: bool = False, ) -> tuple[str, bytes]: rule_runner.set_options( ["--source-root-patterns=src/python", *(extra_args or ())], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) target = rule_runner.get_target(addr) - built_asset = rule_runner.request(BuiltPackage, [PythonAwsLambdaFieldSet.create(target)]) + if layer: + field_set: type[FieldSet] = PythonAwsLambdaLayerFieldSet + else: + field_set = PythonAwsLambdaFieldSet + built_asset = rule_runner.request(BuiltPackage, [field_set.create(target)]) assert expected_extra_log_lines == built_asset.artifacts[0].extra_log_lines digest_contents = rule_runner.request(DigestContents, [built_asset.digest]) assert len(digest_contents) == 1 @@ -328,3 +340,82 @@ def handler(event, context): assert ( zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler" ) + + +def test_create_hello_world_layer(rule_runner: PythonRuleRunner) -> None: + rule_runner.write_files( + { + "src/python/foo/bar/hello_world.py": dedent( + """ + import mureq + + def handler(event, context): + print('Hello, World!') + """ + ), + "src/python/foo/bar/BUILD": dedent( + """ + python_requirement(name="mureq", requirements=["mureq==0.2"]) + python_sources() + + python_aws_lambda_layer( + name='lambda', + dependencies=["./hello_world.py"], + runtime="python3.7", + ) + python_aws_lambda_layer( + name='slimlambda', + include_sources=False, + dependencies=["./hello_world.py"], + runtime="python3.7", + ) + """ + ), + } + ) + + zip_file_relpath, content = create_python_awslambda( + rule_runner, + Address("src/python/foo/bar", target_name="lambda"), + expected_extra_log_lines=(), + layer=True, + ) + assert "src.python.foo.bar/lambda.zip" == zip_file_relpath + + zipfile = ZipFile(BytesIO(content)) + names = set(zipfile.namelist()) + assert "python/mureq/__init__.py" in names + assert "python/foo/bar/hello_world.py" in names + # nothing that looks like a synthesized handler in any of the names + assert "lambda_function.py" not in " ".join(names) + + zip_file_relpath, content = create_python_awslambda( + rule_runner, + Address("src/python/foo/bar", target_name="slimlambda"), + expected_extra_log_lines=(), + layer=True, + ) + assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath + + zipfile = ZipFile(BytesIO(content)) + names = set(zipfile.namelist()) + assert "python/mureq/__init__.py" in names + assert "python/foo/bar/hello_world.py" not in names + # nothing that looks like a synthesized handler in any of the names + assert "lambda_function.py" not in " ".join(names) + + +def test_layer_must_have_dependencies(rule_runner: PythonRuleRunner) -> None: + """A layer _must_ use 'dependencies', unlike most other targets.""" + rule_runner.write_files( + {"BUILD": "python_aws_lambda_layer(name='lambda', runtime='python3.7')"} + ) + with pytest.raises( + ExecutionError, match="The 'dependencies' field in target //:lambda must be defined" + ): + create_python_awslambda( + rule_runner, + Address("", target_name="lambda"), + expected_extra_log_lines=(), + layer=True, + ) diff --git a/src/python/pants/backend/awslambda/python/target_types.py b/src/python/pants/backend/awslambda/python/target_types.py index 3a8a54b4832..84b1795509f 100644 --- a/src/python/pants/backend/awslambda/python/target_types.py +++ b/src/python/pants/backend/awslambda/python/target_types.py @@ -1,9 +1,11 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + import re from dataclasses import dataclass -from typing import Match, Optional, Tuple, cast +from typing import ClassVar, Match, Optional, Tuple, cast from pants.backend.python.target_types import PexCompletePlatformsField, PythonResolveField from pants.backend.python.util_rules.faas import ( @@ -20,6 +22,7 @@ from pants.engine.target import ( COMMON_TARGET_FIELDS, BoolField, + Field, InvalidFieldException, InvalidTargetException, Target, @@ -63,8 +66,20 @@ class PythonAwsLambdaIncludeRequirements(BoolField): default = True help = help_text( """ - Whether to resolve requirements and include them in the Pex. This is most useful with Lambda - Layers to make code uploads smaller when deps are in layers. + Whether to resolve requirements and include them in the AWS Lambda artifact. This is most useful with Lambda + Layers to make code uploads smaller when third-party requirements are in layers. + https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html + """ + ) + + +class PythonAwsLambdaIncludeSources(BoolField): + alias = "include_sources" + default = True + help = help_text( + """ + Whether to resolve first party sources and include them in the AWS Lambda artifact. This is + most useful to allow creating a Lambda Layer with only third-party requirements. https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html """ ) @@ -109,26 +124,20 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]: return int(mo.group("major")), int(mo.group("minor")) -class PythonAWSLambda(Target): - alias = "python_awslambda" - core_fields = ( +class PythonAwsLambdaLayerDependenciesField(PythonFaaSDependencies): + required = True + + +class _AWSLambdaBaseTarget(Target): + core_fields: ClassVar[tuple[type[Field], ...]] = ( *COMMON_TARGET_FIELDS, OutputPathField, - PythonFaaSDependencies, - PythonAwsLambdaHandlerField, PythonAwsLambdaIncludeRequirements, PythonAwsLambdaRuntime, PythonFaaSCompletePlatforms, PythonResolveField, EnvironmentField, ) - help = help_text( - f""" - A self-contained Python function suitable for uploading to AWS Lambda. - - See {doc_url('awslambda-python')}. - """ - ) def validate(self) -> None: if self[PythonAwsLambdaRuntime].value is None and not self[PexCompletePlatformsField].value: @@ -143,6 +152,39 @@ def validate(self) -> None: ) +class PythonAWSLambda(_AWSLambdaBaseTarget): + # TODO: rename to python_aws_lambda_function + alias = "python_awslambda" + core_fields = ( + *_AWSLambdaBaseTarget.core_fields, + PythonFaaSDependencies, + PythonAwsLambdaHandlerField, + ) + help = help_text( + f""" + A self-contained Python function suitable for uploading to AWS Lambda. + + See {doc_url('awslambda-python')}. + """ + ) + + +class PythonAWSLambdaLayer(_AWSLambdaBaseTarget): + alias = "python_aws_lambda_layer" + core_fields = ( + *_AWSLambdaBaseTarget.core_fields, + PythonAwsLambdaIncludeSources, + PythonAwsLambdaLayerDependenciesField, + ) + help = help_text( + f""" + A Python layer suitable for uploading to AWS Lambda. + + See {doc_url('awslambda-python')}. + """ + ) + + def rules(): return ( *collect_rules(), diff --git a/src/python/pants/backend/google_cloud_function/python/rules.py b/src/python/pants/backend/google_cloud_function/python/rules.py index 783c6700898..165f78d89a6 100644 --- a/src/python/pants/backend/google_cloud_function/python/rules.py +++ b/src/python/pants/backend/google_cloud_function/python/rules.py @@ -80,6 +80,7 @@ async def package_python_google_cloud_function( handler=field_set.handler, output_path=field_set.output_path, include_requirements=True, + include_sources=True, reexported_handler_module=PythonGoogleCloudFunctionHandlerField.reexported_handler_module, log_only_reexported_handler_func=True, ), diff --git a/src/python/pants/backend/python/util_rules/faas.py b/src/python/pants/backend/python/util_rules/faas.py index df051977038..3709f3bd5bd 100644 --- a/src/python/pants/backend/python/util_rules/faas.py +++ b/src/python/pants/backend/python/util_rules/faas.py @@ -42,6 +42,7 @@ from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, OutputPathField from pants.engine.addresses import Address, UnparsedAddressInputs from pants.engine.fs import ( + EMPTY_DIGEST, CreateDigest, Digest, FileContent, @@ -401,15 +402,18 @@ class BuildPythonFaaSRequest: target_name: str complete_platforms: PythonFaaSCompletePlatforms - handler: PythonFaaSHandlerField + handler: None | PythonFaaSHandlerField output_path: OutputPathField runtime: PythonFaaSRuntimeField include_requirements: bool + include_sources: bool - reexported_handler_module: str + reexported_handler_module: None | str log_only_reexported_handler_func: bool = False + prefix_in_artifact: None | str = None + @rule async def build_python_faas( @@ -426,32 +430,46 @@ async def build_python_faas( "--resolve-local-platforms", ) - complete_platforms, handler = await MultiGet( - Get(CompletePlatforms, PythonFaaSCompletePlatforms, request.complete_platforms), - Get(ResolvedPythonFaaSHandler, ResolvePythonFaaSHandlerRequest(request.handler)), + complete_platforms_get = Get( + CompletePlatforms, PythonFaaSCompletePlatforms, request.complete_platforms ) + if request.handler: + complete_platforms, handler = await MultiGet( + complete_platforms_get, + Get(ResolvedPythonFaaSHandler, ResolvePythonFaaSHandlerRequest(request.handler)), + ) + else: + complete_platforms = await complete_platforms_get + handler = None # TODO: improve diagnostics if there's more than one platform/complete_platform - # synthesise a source file that gives a fixed handler path, no matter what the entry point is: - # some platforms require a certain name (e.g. GCF), and even on others, giving a fixed name - # means users don't need to duplicate the entry_point config in both the pants BUILD file and - # infrastructure definitions (the latter can always use the same names, for every lambda). - reexported_handler_file = f"{request.reexported_handler_module}.py" - reexported_handler_func = "handler" - reexported_handler_content = ( - f"from {handler.module} import {handler.func} as {reexported_handler_func}" - ) - additional_sources = await Get( - Digest, - CreateDigest([FileContent(reexported_handler_file, reexported_handler_content.encode())]), - ) + if request.reexported_handler_module and handler: + # synthesise a source file that gives a fixed handler path, no matter what the entry point is: + # some platforms require a certain name (e.g. GCF), and even on others, giving a fixed name + # means users don't need to duplicate the entry_point config in both the pants BUILD file and + # infrastructure definitions (the latter can always use the same names, for every lambda). + reexported_handler_file = f"{request.reexported_handler_module}.py" + reexported_handler_func = "handler" + reexported_handler_content = ( + f"from {handler.module} import {handler.func} as {reexported_handler_func}" + ) + additional_sources = await Get( + Digest, + CreateDigest( + [FileContent(reexported_handler_file, reexported_handler_content.encode())] + ), + ) + else: + additional_sources = EMPTY_DIGEST + reexported_handler_func = None repository_filename = "faas_repository.pex" pex_request = PexFromTargetsRequest( addresses=[request.address], internal_only=False, include_requirements=request.include_requirements, + include_source_files=request.include_sources, output_filename=repository_filename, platforms=pex_platforms, complete_platforms=complete_platforms, @@ -473,19 +491,24 @@ async def build_python_faas( layout=PexVenvLayout.FLAT_ZIPPED, platforms=pex_platforms, complete_platforms=complete_platforms, + prefix=request.prefix_in_artifact, output_path=Path(output_filename), description=f"Build {request.target_name} artifact for {request.address}", ), ) - if request.log_only_reexported_handler_func: - handler_text = reexported_handler_func + if reexported_handler_func is None: + handler_log_lines = [] else: - handler_text = f"{request.reexported_handler_module}.{reexported_handler_func}" + if request.log_only_reexported_handler_func: + handler_text = reexported_handler_func + else: + handler_text = f"{request.reexported_handler_module}.{reexported_handler_func}" + handler_log_lines = [f" Handler: {handler_text}"] artifact = BuiltPackageArtifact( output_filename, - extra_log_lines=(f" Handler: {handler_text}",), + extra_log_lines=tuple(handler_log_lines), ) return BuiltPackage(digest=result.digest, artifacts=(artifact,)) diff --git a/src/python/pants/backend/python/util_rules/pex_venv.py b/src/python/pants/backend/python/util_rules/pex_venv.py index faf0a27ce8c..998050ba6fa 100644 --- a/src/python/pants/backend/python/util_rules/pex_venv.py +++ b/src/python/pants/backend/python/util_rules/pex_venv.py @@ -1,5 +1,7 @@ # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + from dataclasses import dataclass from enum import Enum from pathlib import Path @@ -28,6 +30,7 @@ class PexVenvRequest: platforms: PexPlatforms = PexPlatforms() complete_platforms: CompletePlatforms = CompletePlatforms() + prefix: None | str = None @dataclass(frozen=True) @@ -73,6 +76,7 @@ async def pex_venv(request: PexVenvRequest) -> PexVenv: f"--dest-dir={dest_dir}", f"--pex-repository={request.pex.name}", f"--layout={request.layout.value}", + *((f"--prefix={request.prefix}",) if request.prefix is not None else ()), # NB. Specifying more than one of these args doesn't make sense for `venv # create`. Incorrect usage will be surfaced as a subprocess failure. *request.platforms.generate_pex_arg_list(), diff --git a/src/python/pants/backend/python/util_rules/pex_venv_test.py b/src/python/pants/backend/python/util_rules/pex_venv_test.py index 22dc529d896..12ebe00337a 100644 --- a/src/python/pants/backend/python/util_rules/pex_venv_test.py +++ b/src/python/pants/backend/python/util_rules/pex_venv_test.py @@ -196,3 +196,23 @@ def test_complete_platforms_should_choose_appropriate_dependencies_when_possible "out/psycopg2/__init__.py", ), ) + + +def test_prefix_should_add_path( + local_pex: Pex, + rule_runner: RuleRunner, +) -> None: + run_and_validate( + rule_runner, + PexVenvRequest( + pex=local_pex, + layout=PexVenvLayout.FLAT, + prefix="some/prefix", + output_path=Path("out/dir"), + description="testing", + ), + check_globs_exist=( + "out/dir/some/prefix/psycopg2/__init__.py", + "out/dir/some/prefix/first/party.py", + ), + )