Skip to content

Commit 9763bbe

Browse files
Michael Brewerheitorlessa
Michael Brewer
andauthored
refactor(idempotent): Change UX to use a config class for non-persistence related features (#306)
* refactor(idempotent): Create a config class * fix: Reanable test * chore: Some refactoring * docs: Update docs * docs(batch): add example on how to integrate with sentry.io (#308) * chore(docs): Update the docs * fix(tests): Fix coverage-html and various update * refactor: Change back to configure * chore: Hide missing code coverage * chore: bump ci * chore: bump ci Co-authored-by: Heitor Lessa <lessa@amazon.co.uk>
1 parent 153567e commit 9763bbe

File tree

11 files changed

+311
-185
lines changed

11 files changed

+311
-185
lines changed

Diff for: Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ test:
2121
poetry run pytest --cache-clear tests/performance
2222

2323
coverage-html:
24-
poetry run pytest --cov-report html
24+
poetry run pytest -m "not perf" --cov-report=html
2525

2626
pr: lint test security-baseline complexity-baseline
2727

Diff for: aws_lambda_powertools/tracing/extensions.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ def aiohttp_trace_config():
88
TraceConfig
99
aiohttp trace config
1010
"""
11-
from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config
11+
from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config # pragma: no cover
1212

13-
aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)"
13+
aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" # pragma: no cover
1414

15-
return aws_xray_trace_config()
15+
return aws_xray_trace_config() # pragma: no cover

Diff for: aws_lambda_powertools/utilities/idempotency/__init__.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55
from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer
66
from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import DynamoDBPersistenceLayer
77

8-
from .idempotency import idempotent
8+
from .idempotency import IdempotencyConfig, idempotent
99

10-
__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent")
10+
__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "IdempotencyConfig")
+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
from typing import Dict
2+
3+
4+
class IdempotencyConfig:
5+
def __init__(
6+
self,
7+
event_key_jmespath: str = "",
8+
payload_validation_jmespath: str = "",
9+
jmespath_options: Dict = None,
10+
raise_on_no_idempotency_key: bool = False,
11+
expires_after_seconds: int = 60 * 60, # 1 hour default
12+
use_local_cache: bool = False,
13+
local_cache_max_items: int = 256,
14+
hash_function: str = "md5",
15+
):
16+
"""
17+
Initialize the base persistence layer
18+
19+
Parameters
20+
----------
21+
event_key_jmespath: str
22+
A jmespath expression to extract the idempotency key from the event record
23+
payload_validation_jmespath: str
24+
A jmespath expression to extract the payload to be validated from the event record
25+
raise_on_no_idempotency_key: bool, optional
26+
Raise exception if no idempotency key was found in the request, by default False
27+
expires_after_seconds: int
28+
The number of seconds to wait before a record is expired
29+
use_local_cache: bool, optional
30+
Whether to locally cache idempotency results, by default False
31+
local_cache_max_items: int, optional
32+
Max number of items to store in local cache, by default 1024
33+
hash_function: str, optional
34+
Function to use for calculating hashes, by default md5.
35+
"""
36+
self.event_key_jmespath = event_key_jmespath
37+
self.payload_validation_jmespath = payload_validation_jmespath
38+
self.jmespath_options = jmespath_options
39+
self.raise_on_no_idempotency_key = raise_on_no_idempotency_key
40+
self.expires_after_seconds = expires_after_seconds
41+
self.use_local_cache = use_local_cache
42+
self.local_cache_max_items = local_cache_max_items
43+
self.hash_function = hash_function

Diff for: aws_lambda_powertools/utilities/idempotency/idempotency.py

+13-4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from typing import Any, Callable, Dict, Optional
66

77
from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
8+
from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
89
from aws_lambda_powertools.utilities.idempotency.exceptions import (
910
IdempotencyAlreadyInProgressError,
1011
IdempotencyInconsistentStateError,
@@ -29,6 +30,7 @@ def idempotent(
2930
event: Dict[str, Any],
3031
context: LambdaContext,
3132
persistence_store: BasePersistenceLayer,
33+
config: IdempotencyConfig = None,
3234
) -> Any:
3335
"""
3436
Middleware to handle idempotency
@@ -43,20 +45,25 @@ def idempotent(
4345
Lambda's Context
4446
persistence_store: BasePersistenceLayer
4547
Instance of BasePersistenceLayer to store data
48+
config: IdempotencyConfig
49+
Configutation
4650
4751
Examples
4852
--------
4953
**Processes Lambda's event in an idempotent manner**
50-
>>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer
54+
>>> from aws_lambda_powertools.utilities.idempotency import (
55+
>>> idempotent, DynamoDBPersistenceLayer, IdempotencyConfig
56+
>>> )
5157
>>>
52-
>>> persistence_store = DynamoDBPersistenceLayer(event_key_jmespath="body", table_name="idempotency_store")
58+
>>> idem_config=IdempotencyConfig(event_key_jmespath="body")
59+
>>> persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency_store")
5360
>>>
54-
>>> @idempotent(persistence_store=persistence_store)
61+
>>> @idempotent(config=idem_config, persistence_store=persistence_layer)
5562
>>> def handler(event, context):
5663
>>> return {"StatusCode": 200}
5764
"""
5865

59-
idempotency_handler = IdempotencyHandler(handler, event, context, persistence_store)
66+
idempotency_handler = IdempotencyHandler(handler, event, context, config or IdempotencyConfig(), persistence_store)
6067

6168
# IdempotencyInconsistentStateError can happen under rare but expected cases when persistent state changes in the
6269
# small time between put & get requests. In most cases we can retry successfully on this exception.
@@ -82,6 +89,7 @@ def __init__(
8289
lambda_handler: Callable[[Any, LambdaContext], Any],
8390
event: Dict[str, Any],
8491
context: LambdaContext,
92+
config: IdempotencyConfig,
8593
persistence_store: BasePersistenceLayer,
8694
):
8795
"""
@@ -98,6 +106,7 @@ def __init__(
98106
persistence_store : BasePersistenceLayer
99107
Instance of persistence layer to store idempotency records
100108
"""
109+
persistence_store.configure(config)
101110
self.persistence_store = persistence_store
102111
self.context = context
103112
self.event = event

Diff for: aws_lambda_powertools/utilities/idempotency/persistence/base.py

+41-46
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,14 @@
99
import warnings
1010
from abc import ABC, abstractmethod
1111
from types import MappingProxyType
12-
from typing import Any, Dict
12+
from typing import Any, Dict, Optional
1313

1414
import jmespath
1515

1616
from aws_lambda_powertools.shared.cache_dict import LRUDict
1717
from aws_lambda_powertools.shared.jmespath_functions import PowertoolsFunctions
1818
from aws_lambda_powertools.shared.json_encoder import Encoder
19+
from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
1920
from aws_lambda_powertools.utilities.idempotency.exceptions import (
2021
IdempotencyInvalidStatusError,
2122
IdempotencyItemAlreadyExistsError,
@@ -107,55 +108,49 @@ class BasePersistenceLayer(ABC):
107108
Abstract Base Class for Idempotency persistence layer.
108109
"""
109110

110-
def __init__(
111-
self,
112-
event_key_jmespath: str = "",
113-
payload_validation_jmespath: str = "",
114-
expires_after_seconds: int = 60 * 60, # 1 hour default
115-
use_local_cache: bool = False,
116-
local_cache_max_items: int = 256,
117-
hash_function: str = "md5",
118-
raise_on_no_idempotency_key: bool = False,
119-
jmespath_options: Dict = None,
120-
) -> None:
111+
def __init__(self):
112+
"""Initialize the defaults """
113+
self.configured = False
114+
self.event_key_jmespath: Optional[str] = None
115+
self.event_key_compiled_jmespath = None
116+
self.jmespath_options: Optional[dict] = None
117+
self.payload_validation_enabled = False
118+
self.validation_key_jmespath = None
119+
self.raise_on_no_idempotency_key = False
120+
self.expires_after_seconds: int = 60 * 60 # 1 hour default
121+
self.use_local_cache = False
122+
self._cache: Optional[LRUDict] = None
123+
self.hash_function = None
124+
125+
def configure(self, config: IdempotencyConfig) -> None:
121126
"""
122-
Initialize the base persistence layer
127+
Initialize the base persistence layer from the configuration settings
123128
124129
Parameters
125130
----------
126-
event_key_jmespath: str
127-
A jmespath expression to extract the idempotency key from the event record
128-
payload_validation_jmespath: str
129-
A jmespath expression to extract the payload to be validated from the event record
130-
expires_after_seconds: int
131-
The number of seconds to wait before a record is expired
132-
use_local_cache: bool, optional
133-
Whether to locally cache idempotency results, by default False
134-
local_cache_max_items: int, optional
135-
Max number of items to store in local cache, by default 1024
136-
hash_function: str, optional
137-
Function to use for calculating hashes, by default md5.
138-
raise_on_no_idempotency_key: bool, optional
139-
Raise exception if no idempotency key was found in the request, by default False
140-
jmespath_options : Dict
141-
Alternative JMESPath options to be included when filtering expr
142-
"""
143-
self.event_key_jmespath = event_key_jmespath
144-
if self.event_key_jmespath:
145-
self.event_key_compiled_jmespath = jmespath.compile(event_key_jmespath)
146-
self.expires_after_seconds = expires_after_seconds
147-
self.use_local_cache = use_local_cache
148-
if self.use_local_cache:
149-
self._cache = LRUDict(max_items=local_cache_max_items)
150-
self.payload_validation_enabled = False
151-
if payload_validation_jmespath:
152-
self.validation_key_jmespath = jmespath.compile(payload_validation_jmespath)
131+
config: IdempotencyConfig
132+
Idempotency configuration settings
133+
"""
134+
if self.configured:
135+
# Prevent being reconfigured multiple times
136+
return
137+
self.configured = True
138+
139+
self.event_key_jmespath = config.event_key_jmespath
140+
if config.event_key_jmespath:
141+
self.event_key_compiled_jmespath = jmespath.compile(config.event_key_jmespath)
142+
self.jmespath_options = config.jmespath_options
143+
if not self.jmespath_options:
144+
self.jmespath_options = {"custom_functions": PowertoolsFunctions()}
145+
if config.payload_validation_jmespath:
146+
self.validation_key_jmespath = jmespath.compile(config.payload_validation_jmespath)
153147
self.payload_validation_enabled = True
154-
self.hash_function = getattr(hashlib, hash_function)
155-
self.raise_on_no_idempotency_key = raise_on_no_idempotency_key
156-
if not jmespath_options:
157-
jmespath_options = {"custom_functions": PowertoolsFunctions()}
158-
self.jmespath_options = jmespath_options
148+
self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key
149+
self.expires_after_seconds = config.expires_after_seconds
150+
self.use_local_cache = config.use_local_cache
151+
if self.use_local_cache:
152+
self._cache = LRUDict(max_items=config.local_cache_max_items)
153+
self.hash_function = getattr(hashlib, config.hash_function)
159154

160155
def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str:
161156
"""
@@ -180,9 +175,9 @@ def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str:
180175
)
181176

182177
if self.is_missing_idempotency_key(data):
183-
warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}")
184178
if self.raise_on_no_idempotency_key:
185179
raise IdempotencyKeyError("No data found to create a hashed idempotency_key")
180+
warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}")
186181

187182
return self._generate_hash(data)
188183

Diff for: aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

+3-5
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ def __init__(
2626
validation_key_attr: str = "validation",
2727
boto_config: Optional[Config] = None,
2828
boto3_session: Optional[boto3.session.Session] = None,
29-
*args,
30-
**kwargs,
3129
):
3230
"""
3331
Initialize the DynamoDB client
@@ -57,9 +55,9 @@ def __init__(
5755
**Create a DynamoDB persistence layer with custom settings**
5856
>>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer
5957
>>>
60-
>>> persistence_store = DynamoDBPersistenceLayer(event_key="body", table_name="idempotency_store")
58+
>>> persistence_store = DynamoDBPersistenceLayer(table_name="idempotency_store")
6159
>>>
62-
>>> @idempotent(persistence_store=persistence_store)
60+
>>> @idempotent(persistence_store=persistence_store, event_key="body")
6361
>>> def handler(event, context):
6462
>>> return {"StatusCode": 200}
6563
"""
@@ -74,7 +72,7 @@ def __init__(
7472
self.status_attr = status_attr
7573
self.data_attr = data_attr
7674
self.validation_key_attr = validation_key_attr
77-
super(DynamoDBPersistenceLayer, self).__init__(*args, **kwargs)
75+
super(DynamoDBPersistenceLayer, self).__init__()
7876

7977
def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord:
8078
"""

0 commit comments

Comments
 (0)