diff --git a/README.md b/README.md index 234ba219a32..1d9f04aec99 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, * **[Logging](https://awslabs.github.io/aws-lambda-powertools-python/core/logger/)** - Structured logging made easier, and decorator to enrich structured logging with key Lambda context details * **[Metrics](https://awslabs.github.io/aws-lambda-powertools-python/core/metrics/)** - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF) * **[Bring your own middleware](https://awslabs.github.io/aws-lambda-powertools-python/utilities/middleware_factory/)** - Decorator factory to create your own middleware to run logic before, and after each Lambda invocation +* **[Parameters utility](https://awslabs.github.io/aws-lambda-powertools-python/utilities/parameters/)** - Retrieve and cache parameter values from Parameter Store, Secrets Manager, or DynamoDB +* **[Validation](https://)** - ### Installation diff --git a/aws_lambda_powertools/validation/__init__.py b/aws_lambda_powertools/validation/__init__.py new file mode 100644 index 00000000000..924f64e70cc --- /dev/null +++ b/aws_lambda_powertools/validation/__init__.py @@ -0,0 +1,2 @@ +"""Validation utility +""" diff --git a/aws_lambda_powertools/validation/schemas.py b/aws_lambda_powertools/validation/schemas.py new file mode 100644 index 00000000000..bcdf082eeb3 --- /dev/null +++ b/aws_lambda_powertools/validation/schemas.py @@ -0,0 +1,56 @@ +from datetime import date, datetime +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, root_validator +from typing_extensions import Literal + + +class DynamoScheme(BaseModel): + ApproximateCreationDateTime: date + Keys: Dict[Literal["id"], Dict[Literal["S"], str]] + NewImage: Optional[Dict[str, Any]] = {} + OldImage: Optional[Dict[str, Any]] = {} + SequenceNumber: str + SizeBytes: int + StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"] + + @root_validator + def check_one_image_exists(cls, values): + newimg, oldimg = values.get("NewImage"), values.get("OldImage") + stream_type = values.get("StreamViewType") + if stream_type == "NEW_AND_OLD_IMAGES" and not newimg and not oldimg: + raise TypeError("DynamoDB streams schema failed validation, missing both new & old stream images") + return values + + +class DynamoRecordSchema(BaseModel): + eventID: str + eventName: Literal["INSERT", "MODIFY", "REMOVE"] + eventVersion: float + eventSource: Literal["aws:dynamodb"] + awsRegion: str + eventSourceARN: str + dynamodb: DynamoScheme + + +class DynamoDBSchema(BaseModel): + Records: List[DynamoRecordSchema] + + +class EventBridgeSchema(BaseModel): + version: str + id: str # noqa: A003,VNE003 + source: str + account: int + time: datetime + region: str + resources: List[str] + detail: Dict[str, Any] + + +class SqsSchema(BaseModel): + todo: str + + +class SnsSchema(BaseModel): + todo: str diff --git a/aws_lambda_powertools/validation/validator.py b/aws_lambda_powertools/validation/validator.py new file mode 100644 index 00000000000..b3ff685007b --- /dev/null +++ b/aws_lambda_powertools/validation/validator.py @@ -0,0 +1,126 @@ +import logging +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict + +from pydantic import BaseModel, ValidationError + +from aws_lambda_powertools.middleware_factory import lambda_handler_decorator +from aws_lambda_powertools.validation.schemas import DynamoDBSchema, EventBridgeSchema + +logger = logging.getLogger(__name__) + + +class Envelope(ABC): + def _parse_user_dict_schema(self, user_event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any: + logger.debug("parsing user dictionary schema") + try: + return inbound_schema_model(**user_event) + except (ValidationError, TypeError): + logger.exception("Valdation exception while extracting user custom schema") + raise + + def _parse_user_json_string_schema(self, user_event: str, inbound_schema_model: BaseModel) -> Any: + logger.debug("parsing user dictionary schema") + try: + return inbound_schema_model.parse_raw(user_event) + except (ValidationError, TypeError): + logger.exception("Valdation exception while extracting user custom schema") + raise + + @abstractmethod + def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any: + return NotImplemented + + +class UserEnvelope(Envelope): + def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any: + try: + return inbound_schema_model(**event) + except (ValidationError, TypeError): + logger.exception("Valdation exception received from input user custom envelope event") + raise + + +class EventBridgeEnvelope(Envelope): + def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any: + try: + parsed_envelope = EventBridgeSchema(**event) + except (ValidationError, TypeError): + logger.exception("Valdation exception received from input eventbridge event") + raise + return self._parse_user_dict_schema(parsed_envelope.detail, inbound_schema_model) + + +class DynamoDBEnvelope(Envelope): + def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any: + try: + parsed_envelope = DynamoDBSchema(**event) + except (ValidationError, TypeError): + logger.exception("Valdation exception received from input dynamodb stream event") + raise + output = [] + for record in parsed_envelope.Records: + parsed_new_image = ( + {} + if not record.dynamodb.NewImage + else self._parse_user_dict_schema(record.dynamodb.NewImage, inbound_schema_model) + ) # noqa: E501 + parsed_old_image = ( + {} + if not record.dynamodb.OldImage + else self._parse_user_dict_schema(record.dynamodb.OldImage, inbound_schema_model) + ) # noqa: E501 + output.append({"new": parsed_new_image, "old": parsed_old_image}) + return output + + +@lambda_handler_decorator +def validator( + handler: Callable[[Dict, Any], Any], + event: Dict[str, Any], + context: Dict[str, Any], + inbound_schema_model: BaseModel, + outbound_schema_model: BaseModel, + envelope: Envelope, +) -> Any: + """Decorator to create validation for lambda handlers events - both inbound and outbound + + As Lambda follows (event, context) signature we can remove some of the boilerplate + and also capture any exception any Lambda function throws or its response as metadata + + Example + ------- + **Lambda function using validation decorator** + + @validator(inbound=inbound_schema_model, outbound=outbound_schema_model) + def handler(parsed_event_model, context): + ... + + Parameters + ---------- + todo add + + Raises + ------ + err + TypeError or pydantic.ValidationError or any exception raised by the lambda handler itself + """ + lambda_handler_name = handler.__name__ + logger.debug("Validating inbound schema") + parsed_event_model = envelope.parse(event, inbound_schema_model) + try: + logger.debug(f"Calling handler {lambda_handler_name}") + response = handler({"orig": event, "custom": parsed_event_model}, context) + logger.debug("Received lambda handler response successfully") + logger.debug(response) + except Exception: + logger.exception(f"Exception received from {lambda_handler_name}") + raise + + try: + logger.debug("Validating outbound response schema") + outbound_schema_model(**response) + except (ValidationError, TypeError): + logger.exception(f"Validation exception received from {lambda_handler_name} response event") + raise + return response diff --git a/poetry.lock b/poetry.lock index a5570d1445c..bcd22e5f2d5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -112,6 +112,17 @@ dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.int docs = ["sphinx", "zope.interface"] tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +[[package]] +category = "dev" +description = "AWS Lambda Context class for type checking and testing" +name = "aws-lambda-context" +optional = false +python-versions = ">= 3.6" +version = "1.1.0" + +[package.extras] +tests = ["flake8 (3.7.8)", "isort (4.3.21)", "mypy (0.720)", "black (19.3b0)", "pytest (5.0.1)", "pytest-cov (2.7.1)", "pre-commit (1.17.0)", "bump2version (0.5.10)"] + [[package]] category = "main" description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." @@ -162,7 +173,7 @@ typed-ast = ">=1.4.0" d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] -category = "dev" +category = "main" description = "The AWS SDK for Python" name = "boto3" optional = false @@ -671,6 +682,24 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.6.0" +[[package]] +category = "main" +description = "Data validation and settings management using python 3.6 type hinting" +name = "pydantic" +optional = false +python-versions = ">=3.6" +version = "1.6.1" + +[package.dependencies] +[package.dependencies.dataclasses] +python = "<3.7" +version = ">=0.6" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] +typing_extensions = ["typing-extensions (>=3.7.2)"] + [[package]] category = "dev" description = "passive checker of Python programs" @@ -816,7 +845,7 @@ security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [[package]] -category = "dev" +category = "main" description = "An Amazon S3 Transfer Manager" name = "s3transfer" optional = false @@ -948,10 +977,13 @@ version = "1.4.2" idna = ">=2.0" multidict = ">=4.0" +[package.dependencies.typing-extensions] +python = "<3.8" +version = ">=3.7.4" + [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" @@ -962,7 +994,8 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "a2760fd5f04b7f1841509fcbcb4ccdaf35d92d1395627787e4a11f391a0597d2" +content-hash = "18607a712e4a4a05de7350ecbcf26327a4fb45bb8609dc7f3d19b7610c2faafc" +lock-version = "1.0" python-versions = "^3.6" [metadata.files] @@ -1008,6 +1041,10 @@ attrs = [ {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, ] +aws-lambda-context = [ + {file = "aws-lambda-context-1.1.0.tar.gz", hash = "sha256:b6a6d3360671b25bb171335c7e6510b22e60c6fd5db99803f478c986d3230220"}, + {file = "aws_lambda_context-1.1.0-py3-none-any.whl", hash = "sha256:d03b16aaf8abac30b71bc5d66ed8edadd8805e0d581f73f1e9b4b171635c817d"}, +] aws-xray-sdk = [ {file = "aws-xray-sdk-2.6.0.tar.gz", hash = "sha256:abf5b90f740e1f402e23414c9670e59cb9772e235e271fef2bce62b9100cbc77"}, {file = "aws_xray_sdk-2.6.0-py2.py3-none-any.whl", hash = "sha256:076f7c610cd3564bbba3507d43e328fb6ff4a2e841d3590f39b2c3ce99d41e1d"}, @@ -1272,6 +1309,25 @@ pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, ] +pydantic = [ + {file = "pydantic-1.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:418b84654b60e44c0cdd5384294b0e4bc1ebf42d6e873819424f3b78b8690614"}, + {file = "pydantic-1.6.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4900b8820b687c9a3ed753684337979574df20e6ebe4227381d04b3c3c628f99"}, + {file = "pydantic-1.6.1-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:b49c86aecde15cde33835d5d6360e55f5e0067bb7143a8303bf03b872935c75b"}, + {file = "pydantic-1.6.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:2de562a456c4ecdc80cf1a8c3e70c666625f7d02d89a6174ecf63754c734592e"}, + {file = "pydantic-1.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f769141ab0abfadf3305d4fcf36660e5cf568a666dd3efab7c3d4782f70946b1"}, + {file = "pydantic-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2dc946b07cf24bee4737ced0ae77e2ea6bc97489ba5a035b603bd1b40ad81f7e"}, + {file = "pydantic-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:36dbf6f1be212ab37b5fda07667461a9219c956181aa5570a00edfb0acdfe4a1"}, + {file = "pydantic-1.6.1-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:1783c1d927f9e1366e0e0609ae324039b2479a1a282a98ed6a6836c9ed02002c"}, + {file = "pydantic-1.6.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cf3933c98cb5e808b62fae509f74f209730b180b1e3c3954ee3f7949e083a7df"}, + {file = "pydantic-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f8af9b840a9074e08c0e6dc93101de84ba95df89b267bf7151d74c553d66833b"}, + {file = "pydantic-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:40d765fa2d31d5be8e29c1794657ad46f5ee583a565c83cea56630d3ae5878b9"}, + {file = "pydantic-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3fa799f3cfff3e5f536cbd389368fc96a44bb30308f258c94ee76b73bd60531d"}, + {file = "pydantic-1.6.1-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:6c3f162ba175678218629f446a947e3356415b6b09122dcb364e58c442c645a7"}, + {file = "pydantic-1.6.1-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:eb75dc1809875d5738df14b6566ccf9fd9c0bcde4f36b72870f318f16b9f5c20"}, + {file = "pydantic-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:530d7222a2786a97bc59ee0e0ebbe23728f82974b1f1ad9a11cd966143410633"}, + {file = "pydantic-1.6.1-py36.py37.py38-none-any.whl", hash = "sha256:b5b3489cb303d0f41ad4a7390cf606a5f2c7a94dcba20c051cd1c653694cb14d"}, + {file = "pydantic-1.6.1.tar.gz", hash = "sha256:54122a8ed6b75fe1dd80797f8251ad2063ea348a03b77218d73ea9fe19bd4e73"}, +] pyflakes = [ {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, @@ -1432,4 +1488,4 @@ yarl = [ zipp = [ {file = "zipp-3.1.0-py3-none-any.whl", hash = "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b"}, {file = "zipp-3.1.0.tar.gz", hash = "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"}, -] +] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7458636cc74..1f4bee17c59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "aws_lambda_powertools" -version = "1.1.3" +version = "1.3.1" description = "Python utilities for AWS Lambda functions including but not limited to tracing, logging and custom metric" authors = ["Amazon Web Services"] classifiers=[ @@ -21,6 +21,8 @@ license = "MIT-0" python = "^3.6" aws-xray-sdk = "^2.5.0" fastjsonschema = "~=2.14.4" +boto3 = "^1.12" +pydantic = "^1.6.1" [tool.poetry.dev-dependencies] coverage = {extras = ["toml"], version = "^5.0.3"} @@ -47,6 +49,7 @@ xenon = "^0.7.0" flake8-eradicate = "^0.3.0" dataclasses = {version = "*", python = "~3.6"} flake8-bugbear = "^20.1.4" +aws_lambda_context = "^1.1.0" [tool.coverage.run] source = ["aws_lambda_powertools"] diff --git a/tests/functional/test_validator.py b/tests/functional/test_validator.py new file mode 100644 index 00000000000..afbdb5242eb --- /dev/null +++ b/tests/functional/test_validator.py @@ -0,0 +1,171 @@ +import io +from http import HTTPStatus +from typing import Any, Dict, Optional + +import pytest +from aws_lambda_context import LambdaContext +from pydantic import BaseModel +from pydantic.error_wrappers import ValidationError + +from aws_lambda_powertools.validation.validator import DynamoDBEnvelope, EventBridgeEnvelope, UserEnvelope, validator + + +class OutboundSchema(BaseModel): + response_code: HTTPStatus + message: str + + +class InboundSchema(BaseModel): + greeting: str + + +@pytest.fixture +def stdout(): + return io.StringIO() + + +@validator(inbound_schema_model=InboundSchema, outbound_schema_model=OutboundSchema, envelope=UserEnvelope()) +def my_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Optional[Any]]: + assert event["custom"] + assert event["orig"] + return {"response_code": 200, "message": "working"} + + +@validator(inbound_schema_model=InboundSchema, outbound_schema_model=OutboundSchema, envelope=UserEnvelope()) +def my_outbound_fail_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Optional[Any]]: + assert event["custom"] + assert event["orig"] + return {"stuff": 200, "message": "working"} + + +def test_ok_inbound_outbound_validation(): + my_handler({"greeting": "hello"}, LambdaContext()) + + +def test_fail_outbound(): + with pytest.raises(ValidationError): + my_outbound_fail_handler({"greeting": "hello"}, LambdaContext()) + + +def test_fail_inbound_validation(): + with pytest.raises(ValidationError): + my_handler({"this_fails": "hello"}, LambdaContext()) + + +class MyMessage(BaseModel): + message: str + messageId: int + + +@validator(inbound_schema_model=MyMessage, outbound_schema_model=OutboundSchema, envelope=DynamoDBEnvelope()) +def dynamodb_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Optional[Any]]: + assert event["custom"] + # first record + assert not event["custom"][0]["old"] + assert event["custom"][0]["new"].message == "hello" + assert event["custom"][0]["new"].messageId == 8 + # second record + assert event["custom"][1]["new"].message == "new_hello" + assert event["custom"][1]["new"].messageId == 88 + assert event["custom"][1]["old"].message == "hello" + assert event["custom"][1]["old"].messageId == 81 + # third record + assert not event["custom"][2]["new"] + assert event["custom"][2]["old"].message == "hello1" + assert event["custom"][2]["old"].messageId == 82 + assert event["orig"] + + return {"response_code": 200, "message": "working"} + + +def test_dynamodb_fail_inbound_validation(): + event = {"greeting": "hello"} + with pytest.raises(ValidationError): + dynamodb_handler(event, LambdaContext()) + + +def test_dynamodb_ok_inbound_outbound_validation(): + event = { + "Records": [ + { + "eventID": "8ae66d82798e8c34ca4a568d6d2ddb75", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-west-1", + "dynamodb": { + "ApproximateCreationDateTime": 1583747504.0, + "Keys": {"id": {"S": "a5890cc9-47c7-4667-928b-f072b93f7acd"}}, + "NewImage": {"message": "hello", "messageId": 8}, + "SequenceNumber": "4722000000000004992628049", + "SizeBytes": 215, + "StreamViewType": "NEW_AND_OLD_IMAGES", + }, + "eventSourceARN": "arn:aws:dynamodb/stream/2020-0", + }, + { + "eventID": "8ae66d82798e83333568d6d2ddb75", + "eventName": "MODIFY", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-west-1", + "dynamodb": { + "ApproximateCreationDateTime": 1583747504.0, + "Keys": {"id": {"S": "a5890cc9-47c7-4667-928b-f072b93f7acd"}}, + "NewImage": {"message": "new_hello", "messageId": 88}, + "OldImage": {"message": "hello", "messageId": 81}, + "SequenceNumber": "4722000000000004992628049", + "SizeBytes": 215, + "StreamViewType": "NEW_AND_OLD_IMAGES", + }, + "eventSourceARN": "arn:aws:dynamodb/stream/2020-0", + }, + { + "eventID": "8ae66d82798e8c34ca4a568d6d2ddb75", + "eventName": "REMOVE", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-west-1", + "dynamodb": { + "ApproximateCreationDateTime": 1583747504.0, + "Keys": {"id": {"S": "a5890cc9-47c7-4667-928b-f072b93f7acd"}}, + "OldImage": {"message": "hello1", "messageId": 82}, + "SequenceNumber": "4722000000000004992628049", + "SizeBytes": 215, + "StreamViewType": "NEW_AND_OLD_IMAGES", + }, + "eventSourceARN": "arn:aws:dynamodb/stream/2020-0", + }, + ] + } + dynamodb_handler(event, LambdaContext()) + + +@validator(inbound_schema_model=MyMessage, outbound_schema_model=OutboundSchema, envelope=EventBridgeEnvelope()) +def eventbridge_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Optional[Any]]: + assert event["custom"] + assert event["custom"].messageId == 8 + assert event["custom"].message == "hello" + assert event["orig"] + return {"response_code": 200, "message": "working"} + + +def test_eventbridge_ok_validation(): + event = { + "version": "0", + "id": "553961c5-5017-5763-6f21-f88d5f5f4b05", + "detail-type": "my func stream event json", + "source": "arn:aws:lambda:eu-west-1:88888888:function:my_func", + "account": "88888888", + "time": "2020-02-11T08:18:09Z", + "region": "eu-west-1", + "resources": ["arn:aws:dynamodb:eu-west-1:88888888:table/stream/2020-02"], + "detail": {"message": "hello", "messageId": 8}, + } + eventbridge_handler(event, LambdaContext()) + + +def test_eventbridge_fail_inbound_validation(): + event = {"greeting": "hello"} + with pytest.raises(ValidationError): + eventbridge_handler(event, LambdaContext())