From c53d504c07d5532706691f4bce5f358a892c703e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Mon, 25 Jul 2022 15:24:56 +0200 Subject: [PATCH 01/26] feat(idempotency): add option to expire inprogress invocations --- .../utilities/idempotency/base.py | 36 +++++++++++++++++- .../utilities/idempotency/config.py | 4 ++ .../utilities/idempotency/persistence/base.py | 25 ++++++++++++- .../idempotency/persistence/dynamodb.py | 30 +++++++++++++-- tests/functional/idempotency/conftest.py | 37 ++++++++++++++----- tests/functional/idempotency/utils.py | 22 ++++++++--- 6 files changed, 134 insertions(+), 20 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 41fbd232ad3..821e2d46f6b 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -1,3 +1,4 @@ +import datetime import logging from copy import deepcopy from typing import Any, Callable, Dict, Optional, Tuple @@ -73,6 +74,7 @@ def __init__( self.data = deepcopy(_prepare_data(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs + self.config = config persistence_store.configure(config, self.function.__name__) self.persistence_store = persistence_store @@ -101,7 +103,9 @@ def _process_idempotency(self): try: # We call save_inprogress first as an optimization for the most common case where no idempotent record # already exists. If it succeeds, there's no need to call get_record. - self.persistence_store.save_inprogress(data=self.data) + self.persistence_store.save_inprogress( + data=self.data, remaining_time_in_millis=self._get_remaining_time_in_millis() + ) except IdempotencyKeyError: raise except IdempotencyItemAlreadyExistsError: @@ -113,6 +117,25 @@ def _process_idempotency(self): return self._get_function_response() + def _get_remaining_time_in_millis(self) -> Optional[int]: + """ + Tries to determine the remaining time available for the current lambda invocation. + + Currently, it only works if the idempotent handler decorator is used, since we need to acess the lambda context. + However, this could be improved if we start storing the lambda context globally during the invocation. + + Returns + ------- + Optional[int] + Remaining time in millis, or None if the remaining time cannot be determined. + """ + + # Look into fn_args to see if we have a lambda context + if self.fn_args and len(self.fn_args) == 2 and getattr(self.fn_args[1], "get_remaining_time_in_millis", None): + return self.fn_args[1].get_remaining_time_in_millis() + + return None + def _get_idempotency_record(self) -> DataRecord: """ Retrieve the idempotency record from the persistence layer. @@ -167,6 +190,17 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] raise IdempotencyInconsistentStateError("save_inprogress and get_record return inconsistent results.") if data_record.status == STATUS_CONSTANTS["INPROGRESS"]: + # This code path will only be triggered if the expires_in_progress option is enabled, and the item + # became expored between the save_inprogress call an dhere + if ( + self.config.expires_in_progress + and data_record.in_progress_expiry_timestamp is not None + and data_record.in_progress_expiry_timestamp < int(datetime.datetime.now().timestamp()) + ): + raise IdempotencyInconsistentStateError( + "item should have been expired in-progress because it already time-outed." + ) + raise IdempotencyAlreadyInProgressError( f"Execution already in progress with idempotency key: " f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}" diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py index 06468cc74a7..d77a6dbfe0e 100644 --- a/aws_lambda_powertools/utilities/idempotency/config.py +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -9,6 +9,7 @@ def __init__( jmespath_options: Optional[Dict] = None, raise_on_no_idempotency_key: bool = False, expires_after_seconds: int = 60 * 60, # 1 hour default + expires_in_progress: bool = False, use_local_cache: bool = False, local_cache_max_items: int = 256, hash_function: str = "md5", @@ -26,6 +27,8 @@ def __init__( Raise exception if no idempotency key was found in the request, by default False expires_after_seconds: int The number of seconds to wait before a record is expired + expires_in_progress: bool, optional + Whether to expire units of work that timed-out during invocation, by default False use_local_cache: bool, optional Whether to locally cache idempotency results, by default False local_cache_max_items: int, optional @@ -38,6 +41,7 @@ def __init__( self.jmespath_options = jmespath_options self.raise_on_no_idempotency_key = raise_on_no_idempotency_key self.expires_after_seconds = expires_after_seconds + self.expires_in_progress = expires_in_progress self.use_local_cache = use_local_cache self.local_cache_max_items = local_cache_max_items self.hash_function = hash_function diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index e6ffea10de8..95f9213dc3e 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -8,6 +8,7 @@ import os import warnings from abc import ABC, abstractmethod +from math import ceil from types import MappingProxyType from typing import Any, Dict, Optional @@ -40,6 +41,7 @@ def __init__( idempotency_key, status: str = "", expiry_timestamp: Optional[int] = None, + in_progress_expiry_timestamp: Optional[int] = None, response_data: Optional[str] = "", payload_hash: Optional[str] = None, ) -> None: @@ -53,6 +55,8 @@ def __init__( status of the idempotent record expiry_timestamp: int, optional time before the record should expire, in seconds + in_progress_expiry_timestamp: int, optional + time before the record should expire while in the INPROGRESS state, in seconds payload_hash: str, optional hashed representation of payload response_data: str, optional @@ -61,6 +65,7 @@ def __init__( self.idempotency_key = idempotency_key self.payload_hash = payload_hash self.expiry_timestamp = expiry_timestamp + self.in_progress_expiry_timestamp = in_progress_expiry_timestamp self._status = status self.response_data = response_data @@ -120,6 +125,7 @@ def __init__(self): self.validation_key_jmespath = None self.raise_on_no_idempotency_key = False self.expires_after_seconds: int = 60 * 60 # 1 hour default + self.expires_in_progress: bool = False self.use_local_cache = False self.hash_function = None @@ -152,6 +158,7 @@ def configure(self, config: IdempotencyConfig, function_name: Optional[str] = No self.payload_validation_enabled = True self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key self.expires_after_seconds = config.expires_after_seconds + self.expires_in_progress = config.expires_in_progress self.use_local_cache = config.use_local_cache if self.use_local_cache: self._cache = LRUDict(max_items=config.local_cache_max_items) @@ -328,7 +335,7 @@ def save_success(self, data: Dict[str, Any], result: dict) -> None: self._save_to_cache(data_record=data_record) - def save_inprogress(self, data: Dict[str, Any]) -> None: + def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Optional[int] = None) -> None: """ Save record of function's execution being in progress @@ -336,6 +343,8 @@ def save_inprogress(self, data: Dict[str, Any]) -> None: ---------- data: Dict[str, Any] Payload + remaining_time_in_millis: Optional[int] + If expiry of in-progress invocations is enabled, this will contain the remaining time available in millis """ data_record = DataRecord( idempotency_key=self._get_hashed_idempotency_key(data=data), @@ -344,6 +353,20 @@ def save_inprogress(self, data: Dict[str, Any]) -> None: payload_hash=self._get_hashed_payload(data=data), ) + if self.expires_in_progress: + if remaining_time_in_millis: + now = datetime.datetime.now() + period = datetime.timedelta(milliseconds=remaining_time_in_millis) + + # It's very important to use math.ceil here. Otherwise, we might return an integer that will be smaller + # than the current time in milliseconds, due to rounding. This will create a scenario where the record + # looks already expired in the store, but the invocation is still running. + timestamp = ceil((now + period).timestamp()) + + data_record.in_progress_expiry_timestamp = timestamp + else: + logger.debug("Expires in progress is enabled but we couldn't determine the remaining time left") + logger.debug(f"Saving in progress record for idempotency key: {data_record.idempotency_key}") if self._retrieve_from_cache(idempotency_key=data_record.idempotency_key): diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 88955738ecc..e293ff838dc 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -12,7 +12,7 @@ IdempotencyItemAlreadyExistsError, IdempotencyItemNotFoundError, ) -from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord +from aws_lambda_powertools.utilities.idempotency.persistence.base import STATUS_CONSTANTS, DataRecord logger = logging.getLogger(__name__) @@ -25,6 +25,7 @@ def __init__( static_pk_value: Optional[str] = None, sort_key_attr: Optional[str] = None, expiry_attr: str = "expiration", + in_progress_expiry_attr: str = "in_progress_expiration", status_attr: str = "status", data_attr: str = "data", validation_key_attr: str = "validation", @@ -47,6 +48,8 @@ def __init__( DynamoDB attribute name for the sort key expiry_attr: str, optional DynamoDB attribute name for expiry timestamp, by default "expiration" + in_progress_expiry_attr: str, optional + DynamoDB attribute name for in-progress expiry timestamp, by default "in_progress_expiration" status_attr: str, optional DynamoDB attribute name for status, by default "status" data_attr: str, optional @@ -85,6 +88,7 @@ def __init__( self.static_pk_value = static_pk_value self.sort_key_attr = sort_key_attr self.expiry_attr = expiry_attr + self.in_progress_expiry_attr = in_progress_expiry_attr self.status_attr = status_attr self.data_attr = data_attr self.validation_key_attr = validation_key_attr @@ -133,6 +137,7 @@ def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: idempotency_key=item[self.key_attr], status=item[self.status_attr], expiry_timestamp=item[self.expiry_attr], + in_progress_expiry_timestamp=item.get(self.in_progress_expiry_attr), response_data=item.get(self.data_attr), payload_hash=item.get(self.validation_key_attr), ) @@ -153,6 +158,9 @@ def _put_record(self, data_record: DataRecord) -> None: self.status_attr: data_record.status, } + if data_record.in_progress_expiry_timestamp is not None: + item[self.in_progress_expiry_attr] = data_record.in_progress_expiry_timestamp + if self.payload_validation_enabled: item[self.validation_key_attr] = data_record.payload_hash @@ -161,9 +169,18 @@ def _put_record(self, data_record: DataRecord) -> None: logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}") self.table.put_item( Item=item, - ConditionExpression="attribute_not_exists(#id) OR #now < :now", - ExpressionAttributeNames={"#id": self.key_attr, "#now": self.expiry_attr}, - ExpressionAttributeValues={":now": int(now.timestamp())}, + ConditionExpression=( + "attribute_not_exists(#id) OR " + "#now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), + ExpressionAttributeNames={ + "#id": self.key_attr, + "#now": self.expiry_attr, + "#in_progress_expiry": self.in_progress_expiry_attr, + "#status": self.status_attr, + }, + ExpressionAttributeValues={":now": int(now.timestamp()), ":status": STATUS_CONSTANTS["INPROGRESS"]}, ) except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") @@ -183,6 +200,11 @@ def _update_record(self, data_record: DataRecord): "#status": self.status_attr, } + if self.expires_in_progress: + update_expression += ", #in_progress_expiry = :in_progress_expiry" + expression_attr_values[":in_progress_expiry"] = data_record.in_progress_expiry_timestamp + expression_attr_names["#in_progress_expiry"] = self.in_progress_expiry_attr + if self.payload_validation_enabled: update_expression += ", #validation_key = :validation_key" expression_attr_values[":validation_key"] = data_record.payload_hash diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 74deecef123..f2321255544 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -108,18 +108,28 @@ def expected_params_update_item_with_validation( }, "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", - "UpdateExpression": "SET #response_data = :response_data, " - "#expiry = :expiry, #status = :status, " - "#validation_key = :validation_key", + "UpdateExpression": ( + "SET #response_data = :response_data, " + "#expiry = :expiry, #status = :status, " + "#validation_key = :validation_key" + ), } @pytest.fixture def expected_params_put_item(hashed_idempotency_key): return { - "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, - "ExpressionAttributeValues": {":now": stub.ANY}, + "ConditionExpression": ( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), + "ExpressionAttributeNames": { + "#id": "id", + "#now": "expiration", + "#in_progress_expiry": "in_progress_expiration", + "#status": "status", + }, + "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"}, "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } @@ -128,9 +138,18 @@ def expected_params_put_item(hashed_idempotency_key): @pytest.fixture def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key): return { - "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, - "ExpressionAttributeValues": {":now": stub.ANY}, + "ConditionExpression": ( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND " + "#in_progress_expiry < :now AND #status = :inprogress)" + ), + "ExpressionAttributeNames": { + "#id": "id", + "#in_progress_expiry": stub.ANY, + "#now": stub.ANY, + "#status": "status", + }, + "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"}, "Item": { "expiration": stub.ANY, "id": hashed_idempotency_key, diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index ca3862a2d8c..24b6d4f2d05 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -16,9 +16,17 @@ def build_idempotency_put_item_stub( ) -> Dict: idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" return { - "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, - "ExpressionAttributeValues": {":now": stub.ANY}, + "ConditionExpression": ( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), + "ExpressionAttributeNames": { + "#id": "id", + "#now": "expiration", + "#in_progress_expiry": "in_progress_expiration", + "#status": "status", + }, + "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"}, "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } @@ -33,7 +41,11 @@ def build_idempotency_update_item_stub( idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" serialized_lambda_response = json_serialize(handler_response) return { - "ExpressionAttributeNames": {"#expiry": "expiration", "#response_data": "data", "#status": "status"}, + "ExpressionAttributeNames": { + "#expiry": "expiration", + "#response_data": "data", + "#status": "status", + }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, @@ -41,5 +53,5 @@ def build_idempotency_update_item_stub( }, "Key": {"id": idempotency_key_hash}, "TableName": "TEST_TABLE", - "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", + "UpdateExpression": ("SET #response_data = :response_data, " "#expiry = :expiry, #status = :status"), } From 90d21fb65db97bc309eaac36082a1842d855608e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 26 Jul 2022 11:06:43 +0200 Subject: [PATCH 02/26] chore(idempotency): make existing tests pass with expire_in_progress --- tests/functional/idempotency/conftest.py | 20 +- .../idempotency/test_idempotency.py | 224 ++++++++++++++++-- tests/functional/idempotency/utils.py | 11 +- 3 files changed, 228 insertions(+), 27 deletions(-) diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index f2321255544..21503b4789e 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -75,8 +75,8 @@ def default_jmespath(): @pytest.fixture -def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key): - return { +def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key, idempotency_config): + params = { "ExpressionAttributeNames": {"#expiry": "expiration", "#response_data": "data", "#status": "status"}, "ExpressionAttributeValues": { ":expiry": stub.ANY, @@ -88,6 +88,13 @@ def expected_params_update_item(serialized_lambda_response, hashed_idempotency_k "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", } + if idempotency_config.expires_in_progress: + params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" + params["ExpressionAttributeValues"][":in_progress_expiry"] = stub.ANY + params["UpdateExpression"] += ", #in_progress_expiry = :in_progress_expiry" + + return params + @pytest.fixture def expected_params_update_item_with_validation( @@ -195,6 +202,7 @@ def idempotency_config(config, request, default_jmespath): return IdempotencyConfig( event_key_jmespath=request.param.get("event_key_jmespath") or default_jmespath, use_local_cache=request.param["use_local_cache"], + expires_in_progress=request.param.get("expires_in_progress") or False, ) @@ -212,6 +220,14 @@ def config_with_validation(config, request, default_jmespath): ) +@pytest.fixture +def config_with_expires_in_progress(config, request, default_jmespath): + return IdempotencyConfig( + event_key_jmespath=default_jmespath, + expires_in_progress=True, + ) + + @pytest.fixture def config_with_jmespath_options(config, request): class CustomFunctions(functions.Functions): diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 40cee10e4f7..af39dedb57f 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -41,7 +41,16 @@ def get_dataclasses_lib(): # Using parametrize to run test twice, with two separate instances of persistence store. One instance with caching # enabled, and one without. -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_already_completed( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -86,7 +95,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_in_progress( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -135,7 +153,14 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_in_progress_with_cache( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -200,7 +225,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_first_execution( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -234,7 +268,14 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_first_execution_cached( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -280,7 +321,14 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True, "event_key_jmespath": "body"}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": True, "event_key_jmespath": "body", "expires_in_progress": True}, + {"use_local_cache": True, "event_key_jmespath": "body", "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_first_execution_event_mutation( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -299,7 +347,9 @@ def test_idempotent_lambda_first_execution_event_mutation( stubber.add_response( "update_item", ddb_response, - build_idempotency_update_item_stub(data=event["body"], handler_response=lambda_response), + build_idempotency_update_item_stub( + data=event["body"], config=idempotency_config, handler_response=lambda_response + ), ) stubber.activate() @@ -314,7 +364,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_expired( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -349,7 +408,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_exception( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -389,7 +457,14 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_with_validation", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( config_with_validation: IdempotencyConfig, @@ -434,7 +509,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_expired_during_request( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -490,7 +574,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_persistence_exception_deleting( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -525,7 +618,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_persistence_exception_updating( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -560,7 +662,16 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_persistence_exception_getting( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -594,7 +705,14 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_with_validation", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, ) def test_idempotent_lambda_first_execution_with_validation( config_with_validation: IdempotencyConfig, @@ -628,7 +746,14 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "config_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_without_jmespath", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, ) def test_idempotent_lambda_with_validator_util( config_without_jmespath: IdempotencyConfig, @@ -710,7 +835,14 @@ def test_data_record_json_to_dict_mapping_when_response_data_none(): assert response_data is None -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_in_progress_never_saved_to_cache( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): @@ -726,7 +858,14 @@ def test_in_progress_never_saved_to_cache( assert persistence_store._cache.get("key") is None -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + ], + indirect=True, +) def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): # GIVEN a persistence_store with use_local_cache = False persistence_store.configure(idempotency_config) @@ -746,7 +885,14 @@ def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_ assert not hasattr("persistence_store", "_cache") -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_delete_from_cache_when_empty( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): @@ -797,7 +943,12 @@ def test_is_missing_idempotency_key(): @pytest.mark.parametrize( - "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True + "idempotency_config", + [ + {"use_local_cache": False, "event_key_jmespath": "body", "expires_in_progress": True}, + {"use_local_cache": False, "event_key_jmespath": "body", "expires_in_progress": False}, + ], + indirect=True, ) def test_default_no_raise_on_missing_idempotency_key( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context @@ -817,7 +968,12 @@ def test_default_no_raise_on_missing_idempotency_key( @pytest.mark.parametrize( - "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True + "idempotency_config", + [ + {"use_local_cache": False, "event_key_jmespath": "[body, x]", "expires_in_progress": True}, + {"use_local_cache": False, "event_key_jmespath": "[body, x]", "expires_in_progress": False}, + ], + indirect=True, ) def test_raise_on_no_idempotency_key( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context @@ -842,7 +998,13 @@ def test_raise_on_no_idempotency_key( { "use_local_cache": False, "event_key_jmespath": "[requestContext.authorizer.claims.sub, powertools_json(body).id]", - } + "expires_in_progress": False, + }, + { + "use_local_cache": False, + "event_key_jmespath": "[requestContext.authorizer.claims.sub, powertools_json(body).id]", + "expires_in_progress": True, + }, ], indirect=True, ) @@ -1095,7 +1257,14 @@ def dummy_handler(event, context): assert len(persistence_store.table.method_calls) == 0 -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_function_duplicates( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): @@ -1212,7 +1381,14 @@ def collect_payment(payment: Payment): assert result == payment.transaction_id -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": False, "expires_in_progress": False}, + ], + indirect=True, +) def test_idempotent_lambda_compound_already_completed( idempotency_config: IdempotencyConfig, persistence_store_compound: DynamoDBPersistenceLayer, diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index 24b6d4f2d05..61111c55412 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -3,6 +3,7 @@ from botocore import stub +from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from tests.functional.utils import json_serialize @@ -35,12 +36,13 @@ def build_idempotency_put_item_stub( def build_idempotency_update_item_stub( data: Dict, handler_response: Dict, + config: IdempotencyConfig, function_name: str = "test-func", handler_name: str = "lambda_handler", ) -> Dict: idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" serialized_lambda_response = json_serialize(handler_response) - return { + params = { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", @@ -55,3 +57,10 @@ def build_idempotency_update_item_stub( "TableName": "TEST_TABLE", "UpdateExpression": ("SET #response_data = :response_data, " "#expiry = :expiry, #status = :status"), } + + if config.expires_in_progress: + params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" + params["ExpressionAttributeValues"][":in_progress_expiry"] = stub.ANY + params["UpdateExpression"] += ", #in_progress_expiry = :in_progress_expiry" + + return params From 9676e13ea33e5d2bd8b512911ffe1c01d8120d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 26 Jul 2022 15:11:40 +0200 Subject: [PATCH 03/26] chore(idempotency): add tests for expires_in_progress --- .../utilities/idempotency/persistence/base.py | 2 +- .../idempotency/persistence/dynamodb.py | 2 +- tests/functional/idempotency/conftest.py | 4 +- .../idempotency/test_idempotency.py | 158 +++++++++++++++++- tests/functional/idempotency/utils.py | 2 +- 5 files changed, 160 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 95f9213dc3e..a2548d18e8b 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -365,7 +365,7 @@ def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Option data_record.in_progress_expiry_timestamp = timestamp else: - logger.debug("Expires in progress is enabled but we couldn't determine the remaining time left") + warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left") logger.debug(f"Saving in progress record for idempotency key: {data_record.idempotency_key}") diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index e293ff838dc..cd790a69e6f 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -180,7 +180,7 @@ def _put_record(self, data_record: DataRecord) -> None: "#in_progress_expiry": self.in_progress_expiry_attr, "#status": self.status_attr, }, - ExpressionAttributeValues={":now": int(now.timestamp()), ":status": STATUS_CONSTANTS["INPROGRESS"]}, + ExpressionAttributeValues={":now": int(now.timestamp()), ":inprogress": STATUS_CONSTANTS["INPROGRESS"]}, ) except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 21503b4789e..548618de75a 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -136,7 +136,7 @@ def expected_params_put_item(hashed_idempotency_key): "#in_progress_expiry": "in_progress_expiration", "#status": "status", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } @@ -156,7 +156,7 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali "#now": stub.ANY, "#status": "status", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": { "expiration": stub.ANY, "id": hashed_idempotency_key, diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index af39dedb57f..04e9ca7ceff 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -1,5 +1,7 @@ import copy +import datetime import sys +import warnings from hashlib import md5 from unittest.mock import MagicMock @@ -10,7 +12,7 @@ from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEventV2, event_source from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig -from aws_lambda_powertools.utilities.idempotency.base import _prepare_data +from aws_lambda_powertools.utilities.idempotency.base import MAX_RETRIES, _prepare_data from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -386,7 +388,7 @@ def test_idempotent_lambda_expired( lambda_context, ): """ - Test idempotent decorator when lambda is called with an event it succesfully handled already, but outside of the + Test idempotent decorator when lambda is called with an event it successfully handled already, but outside of the expiry window """ @@ -529,7 +531,7 @@ def test_idempotent_lambda_expired_during_request( lambda_context, ): """ - Test idempotent decorator when lambda is called with an event it succesfully handled already. Persistence store + Test idempotent decorator when lambda is called with an event it successfully handled already. Persistence store returns inconsistent/rapidly changing result between put_item and get_item calls. """ @@ -804,6 +806,156 @@ def lambda_handler(event, context): stubber.deactivate() +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": True}, + ], + indirect=True, +) +def test_idempotent_lambda_expires_in_progress_before_expire( + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event, + timestamp_future, + lambda_response, + hashed_idempotency_key, + lambda_context, +): + """ + Test idempotent decorator when expires_in_progress is on and the event is still in progress, before the + lambda expiration window. + """ + + stubber = stub.Stubber(persistence_store.table.meta.client) + + stubber.add_client_error("put_item", "ConditionalCheckFailedException") + + now = datetime.datetime.now() + period = datetime.timedelta(seconds=5) + timestamp_expires_in_progress = str(int((now + period).timestamp())) + + expected_params_get_item = { + "TableName": TABLE_NAME, + "Key": {"id": hashed_idempotency_key}, + "ConsistentRead": True, + } + ddb_response_get_item = { + "Item": { + "id": {"S": hashed_idempotency_key}, + "expiration": {"N": timestamp_future}, + "in_progress_expiration": {"N": timestamp_expires_in_progress}, + "data": {"S": '{"message": "test", "statusCode": 200'}, + "status": {"S": "INPROGRESS"}, + } + } + stubber.add_response("get_item", ddb_response_get_item, expected_params_get_item) + + stubber.activate() + + @idempotent(config=idempotency_config, persistence_store=persistence_store) + def lambda_handler(event, context): + return lambda_response + + with pytest.raises(IdempotencyAlreadyInProgressError): + lambda_handler(lambda_apigw_event, lambda_context) + + stubber.assert_no_pending_responses() + stubber.deactivate() + + +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": True}, + ], + indirect=True, +) +def test_idempotent_lambda_expires_in_progress_after_expire( + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event, + timestamp_future, + lambda_response, + hashed_idempotency_key, + lambda_context, +): + stubber = stub.Stubber(persistence_store.table.meta.client) + + for _ in range(MAX_RETRIES + 1): + stubber.add_client_error("put_item", "ConditionalCheckFailedException") + + one_second_ago = datetime.datetime.now() - datetime.timedelta(seconds=1) + expected_params_get_item = { + "TableName": TABLE_NAME, + "Key": {"id": hashed_idempotency_key}, + "ConsistentRead": True, + } + ddb_response_get_item = { + "Item": { + "id": {"S": hashed_idempotency_key}, + "expiration": {"N": timestamp_future}, + "in_progress_expiration": {"N": str(int(one_second_ago.timestamp()))}, + "data": {"S": '{"message": "test", "statusCode": 200'}, + "status": {"S": "INPROGRESS"}, + } + } + stubber.add_response("get_item", ddb_response_get_item, expected_params_get_item) + + stubber.activate() + + @idempotent(config=idempotency_config, persistence_store=persistence_store) + def lambda_handler(event, context): + return lambda_response + + with pytest.raises(IdempotencyInconsistentStateError): + lambda_handler(lambda_apigw_event, lambda_context) + + stubber.assert_no_pending_responses() + stubber.deactivate() + + +@pytest.mark.parametrize( + "idempotency_config", + [ + {"use_local_cache": False, "expires_in_progress": True}, + {"use_local_cache": True, "expires_in_progress": True}, + ], + indirect=True, +) +def test_idempotent_lambda_expires_in_progress_unavailable_remaining_time( + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event, + lambda_response, + lambda_context, + expected_params_put_item, + expected_params_update_item, +): + stubber = stub.Stubber(persistence_store.table.meta.client) + + ddb_response = {} + stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response("update_item", ddb_response, expected_params_update_item) + + stubber.activate() + + @idempotent(config=idempotency_config, persistence_store=persistence_store) + def lambda_handler(event, context): + return lambda_response + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("default") + lambda_handler(lambda_apigw_event, lambda_context) + assert len(w) == 1 + assert str(w[-1].message) == "Expires in progress is enabled but we couldn't determine the remaining time left" + + stubber.assert_no_pending_responses() + stubber.deactivate() + + def test_data_record_invalid_status_value(): data_record = DataRecord("key", status="UNSUPPORTED_STATUS") with pytest.raises(IdempotencyInvalidStatusError) as e: diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index 61111c55412..b1aa5b2bbf0 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -27,7 +27,7 @@ def build_idempotency_put_item_stub( "#in_progress_expiry": "in_progress_expiration", "#status": "status", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } From 550e4667949ef6fa19f9aa938ac26fc5059aeda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 26 Jul 2022 20:34:53 +0200 Subject: [PATCH 04/26] chore(docs): added docs about `expires_in_progress` --- docs/utilities/idempotency.md | 54 ++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index a5ed14b9150..f0993f93032 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -22,6 +22,7 @@ times with the same parameters**. This makes idempotent operations safe to retry * Ensure Lambda handler returns the same result when called with the same payload * Select a subset of the event as the idempotency key using JMESPath expressions * Set a time window in which records with the same payload should be considered duplicates +* Optionally expires in-progress executions after the Lambda handler timeout ## Getting started @@ -402,13 +403,14 @@ def call_external_service(data: dict, **kwargs): This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). -```python hl_lines="5-9" title="Customizing DynamoDBPersistenceLayer to suit your table structure" +```python hl_lines="5-10" title="Customizing DynamoDBPersistenceLayer to suit your table structure" from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", key_attr="idempotency_key", expiry_attr="expires_at", + in_progress_expiry_attr="in_progress_expires_at", status_attr="current_status", data_attr="result_data", validation_key_attr="validation_key", @@ -422,6 +424,7 @@ Parameter | Required | Default | Description **table_name** | :heavy_check_mark: | | Table name to store state **key_attr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) **expiry_attr** | | `expiration` | Unix timestamp of when record expires +**in_progress_expiry_attr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) **status_attr** | | `status` | Stores status of the lambda execution during and after invocation **data_attr** | | `data` | Stores results of successfully executed Lambda handlers **validation_key_attr** | | `validation` | Hashed representation of the parts of the event used for validation @@ -440,6 +443,7 @@ Parameter | Default | Description **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired +**expires_in_progress** | `False`| Enables expiry of invocations that time out during execution **use_local_cache** | `False` | Whether to locally cache idempotency results **local_cache_max_items** | 256 | Max number of items to store in local cache **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library. @@ -853,6 +857,54 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer): For example, the `_put_record` method needs to raise an exception if a non-expired record already exists in the data store with a matching key. +### Expiring in-progress invocations + +The default behavior when a Lambda invocation times out is for the event to stay locked until `expire_seconds` have passed. Powertools +has no way of knowing if it's safe to retry the operation in this scenario, so we assume the safest approach: to not +retry the operation. + +However, certain types of invocation have less strict requirements, and can benefit from faster expiry of invocations. Ideally, we +can make an in-progress invocation expire as soon as the [Lambda invocation expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/). + +When using this option, powertools will calculate the remaining available time for the invocation, and save it on the idempotency record. +This way, if a second invocation happens after this timestamp, and the record is stil marked `INPROGRESS`, we execute the invocation again +as if it was already expired. This means that if an invocation expired during execution, it will be quickly executed again on the next try. + +This setting introduces no change on the regular behavior where if an invocation succeeds, the results are cached for `expire_seconds` seconds. + +???+ warning "Warning" + Consider whenever you really want this behavior. Powertools can't make any garantee on which state your application was + when it time outed. Ensure that your business logic can be retried at any stage. + +???+ info "Info: Calculating the remaining available time" + For now this only works with the `idempotent` decorator. At the moment we don't have access to the Lambda context when using + the `idempotent_function` so enabling this option is a no-op in that scenario. + +To activate this behaviour, enable the `expires_in_progress` option on the configuration: + +=== "app.py" + + ```python hl_lines="7" + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + @idempotent(persistence_store=persistence_layer, expires_in_progress=True) + def handler(event, context): + payment = create_subscription_payment( + user=event['user'], + product=event['product_id'] + ) + ... + return { + "payment_id": payment.id, + "message": "success", + "statusCode": 200, + } + ``` + ## Compatibility with other utilities ### Validation utility From bf18cc20e699eb1bf50cf9e4d19e643ac1dc1266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 09:38:50 +0200 Subject: [PATCH 05/26] chore(idempotency): refactored expire in-progress logic --- .../idempotency/persistence/dynamodb.py | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index cd790a69e6f..5e1bf38e556 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -167,20 +167,36 @@ def _put_record(self, data_record: DataRecord) -> None: now = datetime.datetime.now() try: logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}") + + condition_expression = "attribute_not_exists(#id) OR #now < :now" + expression_attribute_names = { + "#id": self.key_attr, + "#now": self.expiry_attr, + "#status": self.status_attr, + } + expression_attribute_values = {":now": int(now.timestamp())} + + # When we want to expire_in_progress invocations, we check if the in_progress timestamp exists + # and we are past that timestamp. We also make sure the status is INPROGRESS because we don't + # want to repeat COMPLETE invocations. + # + # We put this in an if block because customers might want to disable the feature, + # reverting to the old behavior that relies just on the idempotency key. + if self.expires_in_progress: + condition_expression += ( + " OR (" + "attribute_exists(#in_progress_expiry) AND " + "#in_progress_expiry < :now AND #status = :inprogress" + ")" + ) + expression_attribute_names["#in_progress_expiry"] = self.in_progress_expiry_attr + expression_attribute_values[":inprogress"] = STATUS_CONSTANTS["INPROGRESS"] + self.table.put_item( Item=item, - ConditionExpression=( - "attribute_not_exists(#id) OR " - "#now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - ), - ExpressionAttributeNames={ - "#id": self.key_attr, - "#now": self.expiry_attr, - "#in_progress_expiry": self.in_progress_expiry_attr, - "#status": self.status_attr, - }, - ExpressionAttributeValues={":now": int(now.timestamp()), ":inprogress": STATUS_CONSTANTS["INPROGRESS"]}, + ConditionExpression=condition_expression, + ExpressionAttributeNames=expression_attribute_names, + ExpressionAttributeValues=expression_attribute_values, ) except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") @@ -200,16 +216,16 @@ def _update_record(self, data_record: DataRecord): "#status": self.status_attr, } - if self.expires_in_progress: - update_expression += ", #in_progress_expiry = :in_progress_expiry" - expression_attr_values[":in_progress_expiry"] = data_record.in_progress_expiry_timestamp - expression_attr_names["#in_progress_expiry"] = self.in_progress_expiry_attr - if self.payload_validation_enabled: update_expression += ", #validation_key = :validation_key" expression_attr_values[":validation_key"] = data_record.payload_hash expression_attr_names["#validation_key"] = self.validation_key_attr + if self.expires_in_progress: + update_expression += ", #in_progress_expiry = :in_progress_expiry" + expression_attr_values[":in_progress_expiry"] = data_record.in_progress_expiry_timestamp + expression_attr_names["#in_progress_expiry"] = self.in_progress_expiry_attr + kwargs = { "Key": self._get_key(data_record.idempotency_key), "UpdateExpression": update_expression, From c88482e2d684943dd21c071bd14fbc89596b6c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 09:41:15 +0200 Subject: [PATCH 06/26] chore(idempotency): add tests --- tests/functional/idempotency/conftest.py | 66 +++++++++++-------- .../idempotency/test_idempotency.py | 34 +++++----- tests/functional/idempotency/utils.py | 24 ++++--- 3 files changed, 72 insertions(+), 52 deletions(-) diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 548618de75a..c1fd5a8a2ef 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -98,9 +98,9 @@ def expected_params_update_item(serialized_lambda_response, hashed_idempotency_k @pytest.fixture def expected_params_update_item_with_validation( - serialized_lambda_response, hashed_idempotency_key, hashed_validation_key + serialized_lambda_response, hashed_idempotency_key, hashed_validation_key, idempotency_config ): - return { + params = { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", @@ -122,41 +122,48 @@ def expected_params_update_item_with_validation( ), } + if idempotency_config.expires_in_progress: + params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" + params["ExpressionAttributeValues"][":in_progress_expiry"] = stub.ANY + params["UpdateExpression"] += ", #in_progress_expiry = :in_progress_expiry" + + return params + @pytest.fixture -def expected_params_put_item(hashed_idempotency_key): - return { - "ConditionExpression": ( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - ), +def expected_params_put_item(hashed_idempotency_key, idempotency_config): + params = { + "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", "ExpressionAttributeNames": { "#id": "id", "#now": "expiration", - "#in_progress_expiry": "in_progress_expiration", "#status": "status", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY}, "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } + if idempotency_config.expires_in_progress: + params[ + "ConditionExpression" + ] += " OR (attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" + params["ExpressionAttributeValues"][":inprogress"] = "INPROGRESS" + + return params + @pytest.fixture -def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key): - return { - "ConditionExpression": ( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND " - "#in_progress_expiry < :now AND #status = :inprogress)" - ), +def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key, idempotency_config): + params = { + "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", "ExpressionAttributeNames": { "#id": "id", - "#in_progress_expiry": stub.ANY, - "#now": stub.ANY, + "#now": "expiration", "#status": "status", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY}, "Item": { "expiration": stub.ANY, "id": hashed_idempotency_key, @@ -166,6 +173,15 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali "TableName": "TEST_TABLE", } + if idempotency_config.expires_in_progress: + params[ + "ConditionExpression" + ] += " OR (attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" + params["ExpressionAttributeValues"][":inprogress"] = "INPROGRESS" + + return params + @pytest.fixture def hashed_idempotency_key(lambda_apigw_event, default_jmespath, lambda_context): @@ -203,6 +219,7 @@ def idempotency_config(config, request, default_jmespath): event_key_jmespath=request.param.get("event_key_jmespath") or default_jmespath, use_local_cache=request.param["use_local_cache"], expires_in_progress=request.param.get("expires_in_progress") or False, + payload_validation_jmespath=request.param.get("payload_validation_jmespath") or "", ) @@ -211,15 +228,6 @@ def config_without_jmespath(config, request): return IdempotencyConfig(use_local_cache=request.param["use_local_cache"]) -@pytest.fixture -def config_with_validation(config, request, default_jmespath): - return IdempotencyConfig( - event_key_jmespath=default_jmespath, - use_local_cache=request.param, - payload_validation_jmespath="requestContext", - ) - - @pytest.fixture def config_with_expires_in_progress(config, request, default_jmespath): return IdempotencyConfig( diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 04e9ca7ceff..631355624d4 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -345,7 +345,11 @@ def test_idempotent_lambda_first_execution_event_mutation( event = copy.deepcopy(lambda_apigw_event) stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, build_idempotency_put_item_stub(data=event["body"])) + stubber.add_response( + "put_item", + ddb_response, + build_idempotency_put_item_stub(data=event["body"], config=idempotency_config), + ) stubber.add_response( "update_item", ddb_response, @@ -459,17 +463,17 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "config_with_validation", + "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, + {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, + {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, + {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, ], indirect=True, ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( - config_with_validation: IdempotencyConfig, + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, @@ -499,7 +503,7 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=config_with_validation, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -707,17 +711,17 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "config_with_validation", + "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, + {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, + {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, + {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, ], indirect=True, ) def test_idempotent_lambda_first_execution_with_validation( - config_with_validation: IdempotencyConfig, + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item_with_validation, @@ -737,7 +741,7 @@ def test_idempotent_lambda_first_execution_with_validation( stubber.add_response("update_item", ddb_response, expected_params_update_item_with_validation) stubber.activate() - @idempotent(config=config_with_validation, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index b1aa5b2bbf0..383a5298bf6 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -13,25 +13,33 @@ def hash_idempotency_key(data: Any): def build_idempotency_put_item_stub( - data: Dict, function_name: str = "test-func", handler_name: str = "lambda_handler" + data: Dict, + config: IdempotencyConfig, + function_name: str = "test-func", + handler_name: str = "lambda_handler", ) -> Dict: idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" - return { - "ConditionExpression": ( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - ), + params = { + "ConditionExpression": ("attribute_not_exists(#id) OR #now < :now"), "ExpressionAttributeNames": { "#id": "id", "#now": "expiration", - "#in_progress_expiry": "in_progress_expiration", "#status": "status", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY}, "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } + if config.expires_in_progress: + params[ + "ConditionExpression" + ] += " OR (attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" + params["ExpressionAttributeValues"][":inprogress"] = "INPROGRESS" + + return params + def build_idempotency_update_item_stub( data: Dict, From f47ab1f23ca43164c1e3122d9ca14b51e80f407d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 09:41:27 +0200 Subject: [PATCH 07/26] chore(idempotency): remove unused fixtures in tests --- .../idempotency/test_idempotency.py | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 631355624d4..6f0164bd145 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -244,9 +244,6 @@ def test_idempotent_lambda_first_execution( expected_params_update_item, expected_params_put_item, lambda_response, - serialized_lambda_response, - deserialized_lambda_response, - hashed_idempotency_key, lambda_context, ): """ @@ -384,11 +381,9 @@ def test_idempotent_lambda_expired( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, - timestamp_expired, lambda_response, expected_params_update_item, expected_params_put_item, - hashed_idempotency_key, lambda_context, ): """ @@ -428,8 +423,6 @@ def test_idempotent_lambda_exception( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, - timestamp_future, - lambda_response, hashed_idempotency_key, expected_params_put_item, lambda_context, @@ -594,9 +587,6 @@ def test_idempotent_persistence_exception_deleting( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, - timestamp_future, - lambda_response, - hashed_idempotency_key, expected_params_put_item, lambda_context, ): @@ -638,9 +628,6 @@ def test_idempotent_persistence_exception_updating( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, - timestamp_future, - lambda_response, - hashed_idempotency_key, expected_params_put_item, lambda_context, ): @@ -682,10 +669,6 @@ def test_idempotent_persistence_exception_getting( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, - timestamp_future, - lambda_response, - hashed_idempotency_key, - expected_params_put_item, lambda_context, ): """ @@ -727,8 +710,6 @@ def test_idempotent_lambda_first_execution_with_validation( expected_params_update_item_with_validation, expected_params_put_item_with_validation, lambda_response, - hashed_idempotency_key, - hashed_validation_key, lambda_context, ): """ @@ -1107,7 +1088,7 @@ def test_is_missing_idempotency_key(): indirect=True, ) def test_default_no_raise_on_missing_idempotency_key( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" function_name = "foo" @@ -1132,7 +1113,7 @@ def test_default_no_raise_on_missing_idempotency_key( indirect=True, ) def test_raise_on_no_idempotency_key( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request persistence_store.configure(idempotency_config) @@ -1165,7 +1146,7 @@ def test_raise_on_no_idempotency_key( indirect=True, ) def test_jmespath_with_powertools_json( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN an event_key_jmespath with powertools_json custom function persistence_store.configure(idempotency_config, "handler") @@ -1186,7 +1167,7 @@ def test_jmespath_with_powertools_json( @pytest.mark.parametrize("config_with_jmespath_options", ["powertools_json(data).payload"], indirect=True) def test_custom_jmespath_function_overrides_builtin_functions( - config_with_jmespath_options: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + config_with_jmespath_options: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN a persistence store with a custom jmespath_options # AND use a builtin powertools custom function From a4f8ce705c15146f8ac5d9dac5a413db94a52029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 09:44:04 +0200 Subject: [PATCH 08/26] chore(idempotency): make mypy happy --- .../utilities/idempotency/persistence/dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 5e1bf38e556..738c32c7126 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -174,7 +174,7 @@ def _put_record(self, data_record: DataRecord) -> None: "#now": self.expiry_attr, "#status": self.status_attr, } - expression_attribute_values = {":now": int(now.timestamp())} + expression_attribute_values: Dict[str, Any] = {":now": int(now.timestamp())} # When we want to expire_in_progress invocations, we check if the in_progress timestamp exists # and we are past that timestamp. We also make sure the status is INPROGRESS because we don't From 975933d63cc7e0ed0e620243604e8981e69b379f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 11:30:12 +0200 Subject: [PATCH 09/26] chores(documentation): update sample code for `expires_in_progress` --- docs/utilities/idempotency.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index f0993f93032..ee16fcba3ba 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -884,14 +884,18 @@ To activate this behaviour, enable the `expires_in_progress` option on the confi === "app.py" - ```python hl_lines="7" + ```python hl_lines="8" from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, idempotent + DynamoDBPersistenceLayer, IdempotencyConfig, idempotent ) persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - @idempotent(persistence_store=persistence_layer, expires_in_progress=True) + config = IdempotencyConfig( + expires_in_progress=True, + ) + + @idempotent(persistence_store=persistence_layer, config=config) def handler(event, context): payment = create_subscription_payment( user=event['user'], From cd90fc16634dccfc0cfbd1255200944256726dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 12:33:09 +0200 Subject: [PATCH 10/26] chore(documentation): replace idempotency diagrams with mermaid.js --- docs/media/idempotent_sequence.png | Bin 74622 -> 0 bytes docs/media/idempotent_sequence_exception.png | Bin 46647 -> 0 bytes docs/utilities/idempotency.md | 44 +++++++++++++++++- .../event_handler/test_api_gateway.py | 2 +- 4 files changed, 43 insertions(+), 3 deletions(-) delete mode 100644 docs/media/idempotent_sequence.png delete mode 100644 docs/media/idempotent_sequence_exception.png diff --git a/docs/media/idempotent_sequence.png b/docs/media/idempotent_sequence.png deleted file mode 100644 index 92593184abbfb79cc673af07bfbb019bf6c06812..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 74622 zcmce;1yq#X_xEkn-5t^?-O{N;hqMyHkb=^s(jW~Y-9z^PBBgW-3?PjlA)V6kobmSl z)j!s=*7L5{<#Gu#b6qF)+55XcdtVc(uBwQIPL6)##tkfGB{|I-H}34+xN#d5^%nRI z)7{nR8#i9xP?nR{b~o8fLvbhUpy=dAL%V~I@43@-JSV1S`FO z3@Tj!F8D_-K={b29T0^L8o5tZ$+Lx=Fi>Y`1U>~hQSwep5TYsW=8-iey2pT zym>!!`|iXe1pV7aZ;$q9*H=q-6cl1lc)Zo#g;$agm?@DSTN6j{WZ<^RLMY4$Vuo7~ z6PM@5D0lCsiF@qN)W6g#GphGFKgv;z8Xb^Uy7lbj&rqYmP&vrKP=kV#1v{DOa~zAG z8P+*qp`xNR_x>Bb9s4rygBLr z-FfZ@06sxEREIn^RuyC?zGOu0E8&uJ^ys>BrRbol@fl;uxpX{pDi4 z^0`kg8&{X-+(sWW9k6BaQQMB&KazU!&;Dkx-Q{0}oGB@5+x z@ZYb!xdp?g&;R{|BVuBH-uiED7HW(zY5eGOx?hNb<690nkx@Ub~r-`W_2%!IHQc#_^nm6JF#D?KVT%b-e?yiKwY_AypXy zP))4Mv-Z$mQw$qNr>hSq_Nu=s*uUQ zgY$mPg33EEs!~R_-?qx|0p@chZawqSq^zt5_fGZmfg93>g_=%kU$vR|S}v;OED#Dk zPojc;^dZ->qLP_OZ952?@j7gYG~9(724f6(@}={mUX2Ya&xHC&wmhe`s{M*P{86Io zcd4|S$Iy3oaJ9LN5C)7>l9A<64?8`O8Rl8ahp(H{fN@hf?*2#UW`4}qhQo4 zyC5Z#k#U9gy$d&P{Me+Hp=;_ZqLPyP_acnKku$tU@gsc*PWtrX;A>RAs{O^;hVt5o z#}__VJ2m9(y^fE}ZdAWoBeai|H>|Po{F=@gNo`yVQL+1%nZJY0*1x1PGhH7oyt?vA z5!8WVhsU;DopEB1*1sgMkL~?E<2>f(T$m+r7E%+W#{n>j7f%BZIZVXS(lztYZTOfo zT{gxEYo@AKDeN`P_<|$G&8&V;HDDMHNg0>gJpbL;PBgX+5Ttz5wOyJUsy7C9kHbhU1&v$H3k-PV*^JFS@GoE-~gt>e72 z4&BJ#BZ3z}+uHoSeIz$#8!VG}1BdI`^tgB&=O#$FvDpv02WxD3JzPSp|LAhJ1OO!2eWF~_d8sVf?49x=9z7v-YR$MiC zwC>fK3~&q^Ls?ntjxR#^NZCq^PWJ{_+n%(y<`V!R=#3E_tDa7bs=oF}97CM(t z*Bw7CkCLP=;Xr>zyoZ&9K{6P@;Xew^;IdJab@4< zH};M*C;B?H`#ZA@m;z)%WqOlUfiW+8<8OXVr_y}xpC-|;e0eOE6D#al?=1g7N=7cB za9mNB4G}qhb=HT|*68_*Ejxf-e*bU{p%OXcFdJqUoFyw(gkJCNBz7_HBLuOVa!&?! zwK-9#@#SrO$sIqWGgP}}Z6K$E6BQrg(-m2YfE7KaV60Jd$1Pm=^1=P^CpD?7?U-}t zJE&Ifi}QtGvYyy_i@x`Gw_h!NQ8#(l_>A;1h867bTDy&om>ZY&`%Hf1vCiHKZ7s1j z*jp$0bgxrWJ>Se>E>$Ea!9#A<;M1$H>S{h`Gtyx3$!e=s+WWTSnAx(SvL(jnp)-#T z7rC2BmseK8#7Q->99#L!yDQ4zkwRDp~|cJI6lAnfS~nb)C^kgsMP`*gL3>tkl!(b#JUFE3BnTcxS!F2EbR60w4 z7|dt)X~uq(qSrAOA5>QeFAd|mH$j}|h&L&4Ia7M8wH3Ry+HNv<_vvSbWBevkuTm2U ztyeucir>se1GBF#5PC}@uYRJ~nLRlEIqcZ_a&PfSB_S+jZX8}fx6UfWb0~f9)at4L zsW)WJ%xAQqQ^k62D?M>o)Tc1^+>*7LkDkzRJygX~!*2--+%g^O$YY}86l%Ms#|xC_Yr$L^(cq7ll4yrch$71 zkQ`u}8Y^gMlZ?8{@DsLR&im{MP(|C8>t;>t19f{;1B+-=GsB9fsXZ5pHM*9?X}k zk#7_SZ+SwgUE2<}>S$D=%6F`GS*-48An^V1?9~XL%Oi>~J)*1Cvu0HChv2<*-@0Rr zW3cYatckeVQHf(wG<$aM?z?bb?R9zdX5YiME2gZ%Bpte!QmhNnTNxgd(A9^N1^gCQ zz@TLizPr+rEbIZ~j*e<}GOf~$bicM|rGD0B%O9}0v6_fWYG+?;CXuGbbLdOu!GJqZ zJ4sL@%!3MW!s*2&FAoz04<0>A)_te*JuzJ|%2yV*lT0}Z*Na8}bcxo-(=j!ECRM`g z&c1e$wvlK2^FCmlQ^U!j26sB?Ar$(iN}t%!*R?)A2^@}W9QK93u=kva0n)P zW$25neu4xGEVRkx9Z8xWrCi4J!4T52Lf(Ixm~S7mJ}-$ve)!3#A9mx z6-Go83W2Ajq!hcqWEjsUOW;`{FmuP3Ld-SPeVd#li(#3Pk1{m~i!B;{5rhe?ND_0? zO@L%Pl-tgxa@9iMw-NkhJVC)IJ|_!O+}t?8GmjM~5}T8sL#M`yty)9hq;Zg?m@TIA zm%=3cd{d0Rm%484pB}6}0JayaUCFb}+Om8L zvS^vmXNxsq^)I`uT=_*T}((x!a!puw}FaFAGx5SSW_!OgploX(8>CHj0%4;DMNdd1lw=ik3HQ;P62=`?gHbTo^4P20HcUCaTtRDf{j-{6 z;$fV0R;_ylD(cDdmH8uZ(yb43z7jgn+IJ<|Fq>$gdWnk9dKmFPxj8=fG(ko>7_L{mD)XsTU8SVc|XQigip%jLN z`OfHCq5d62yR%#A)f-NUy*wRA3Rjxm6x?K=E~rUh3%K3KVIJ0I*;BRlsq>&+D&?XnlBQ z{UG(k^fq(4q#t%Yn{$a8-ydE54(4<6qUD7B>2fb4m#J?}L0W~?SL|o0Y$y%Cr@8<_ z1O4OsPx{EDTjOayjbb;{Bp;pZb=ROLAPlnqQoC!R+@&@KhvY&+di6ugkyNI1R^ugP z@gDHBK39mCW$z8v4?v*d&YMU3*RLUhY1IajEj#czehveti&vA7tm}O*2k1XD0ueD+ zI|h~aT~*e9(gUz67!;!LoSw$x;W z|NA>K*^U*U{6eCRySa(b+E9l&I>>@@^{eOfm^!xkb9jF%s#fc%#1%K@_7q1D^`~oW zU-Ph($OZ>3a$?uNZa!+k+9F5~#9$T{6U$dkEAqP@XR!c#TVs=;<(sc4wmEJywYy8J zkuhGnVfMgheN@5HH;;!G-}tGy8X+6rh=zBP{?y8Vsq6aagoSTjk!j2LZYB*8H}&cG zXE=1H2j{bmSmIW)pkn0XJQX6jjI;+?YGp?HL*WU2qg@}>Unw&P*c;p)+dVoW{J~HN z%?jvnBcTu|8fkFr19m%BsP(0%ht;%sGeyuRt0bZKcdjSc*<>xSm800AQ%)_SbBD(U zWLBtJKIhHhEDZTSY``kl+NW%+B^803k5uU$`52W3yJasS(31p(29Oj1+k2F#!BW0^ zrM%1SacN>C$^G-8q>|0YB&Ln{8AC}0XcsOUy3ywO)~5VAP=g6INgu{$HNzZ*B)t2d zUV)%1Q!tu39Qpf1&Nh(TC;Sm&9TrJN;o;$tS<7@l@;af9wQbHLU-ZJ+g#}tEHc;>m z2_xd3CLiT_^ozb+c~BYfnii5g8sb2TCFye(-0eRR`~|*0FkNZx)tX~qiau!oFJL52 z=!s*dc(qpE;>)gOSW}cksn&M1bB!IyaxH#3a&c2AKsDey@Qm>d#qW^GG=-O$|Gfkq z8(o{J^W)>A^|4#oJRq2M8(^*+Z5c-5kqYj~ZkBBsfi$Fjd;hB@_kb*c`-8@~^mGYB zDOa;Qd=^TliT_PCfWyb4o}s3bXc+$n^688_I``2C$%@?M{;ruMs}m@$=}9(cbE>86OdUkn~Gk z*JpuS0<*zgr1i-=7%!CwDUc6a`l9jRh0H~&;Fd(vCy<9yT6RU&HRFuJAQC^7^H%dt zJvJM5#-){ly*ag6gceJk3q3ZF0*qqmd`EIHMTjDs$7UM@?yb=A*RBcSOsL3&g6UxK z$rP`-Q?dpb5(F{R36zdJFseP_sgtvQ_fC$ay^NU&56QIvELs~3!r8IYEWx&9{wvQX zh7{lbPv(J01m5o_h$?WhC0cX!sd}k9Kd6N)WWgdTf1k2{VN+{; z?>SDNf&@&+@hTeb!mWCi>0iH@k0WtHxhEKzcjmKKyn*@4VXSk@Nxo+-52> zGxPQ98>Xi70K55GslTuhmQH0-J?c$hUuU9az>6jXbn_{&754|*F$GV-#oZ_L5b5F` zybm4>&CmN?p8X1;?ZZM*Mb8f+XooIZ$jZo!LZp=Fe>1b+=_BcJh&)ocFdU7+R|nHD z(9x%=t#HhO%gc9c+TO&q7B1XzN*12{O&4EB%e`Lk`s^ppJw7Gn zy8=8?Qc_&pim?~pbN?PHK!3l7`rpqFT=u?6s+imMQcoNz&G}YOy1ke#&e@IYB7#k{ ze~g+&!ZX+5mJI&Q$j#7kDjjL_+nrY^VW7=}Ols-V(g2fazeHFGb3qR~bAU&iCgP-5QZrfc6t(rikU zb5^Il011j8p*m4+<_F*|J|%TWqYLU1PCnZEW$5R7 zf_0_Cj4OkW_3tmfM?!Oe#FaGtlmGsIhzat#f4z{chaAOUKl~)JG58y;dj|f?+j$~k z%bLFe78Sntok$h`LH?C(CW@f5v$JV`&jYvN=0f3k_VBY}nEV>k-!p1=ka91o zv3}M57$GVml1Il-o2!~;Fa!I0A=;&r1&u$wazhQ}Y`NIS*v`((r2Vxe|5xomc?z{h z8S)5U;2%YXa3x5C-?gt&P#m*bInh1A{b#eYNpfMc zpImGK{C@hBHbC~piw724fc1~n6A5c0Oqx7@eMq7ZawtG#Gx76lZ@TO)%DW`L1u9w5 z!aq_kU;fM6yBM93RAT0xm@c;}aYo@ws_MI-LUB>7N*jP{MuW5kPiY&F>FAxGot>SZ z?@U&sOf7&KLbUt(3_4{IG?7E!%y;J--06pci^1r{)%lKDIf&cE2*SqWIWPKYkO_@J zHPmZtbe6s*@#TxYnLx2IXw4$}+ZIitlU)IcX9qQj7GDVq`=?L8gO4K}K|Rq=9mHY@ zZ3L)R&`XTMbw7Ha?mMZ{CzSE_})Uq5u|AUf-4BbCeWa zKBUBJ$gx7rj0c&{dqgs9w0%R=5}T7<%Zd0= zBA5IEz~iE?eny+M=%q^fnKV{~hdXVIm)=*5%w$=f1Qi_3_^6kJ=`U9Y(V~u^e5qPM zwemwpg0Lvmli7#_4i64ecrDSfnUm$M&zroC#|yO-`4d@0!1mXaw*hEHX@K|V1pK_~ zV{NcCV_wK6WIwgEw#KTNmYPb;X~1x-IUqqT;+#7XUt-#F75RLdhu@?~yKrvMdL5A; z#c>pgm8|<7jcjdeD}sj80FJ9{y$G_+zF1=;qlafd?=LrBU0p#(a%0F;64-P;y@Fn_ z06o`#JAzFUhfpBi&7r@OEF%B{9EY~ zUQ;!;%;M%Z5%br@>DX_cwkd#`_ zU+c2L*nxh>+p}e5WxHC#RJ14BY4J0$2|L-6iDexAfBC|zn$^Zm_wt@}2Wm@6S;0RbiI-9JUn zcJ#AxI2qsbcywJZYX;Nsh;W#M%R1qd*%TZKEQ`1hm}^2|t*t?P&SAKI*HFP?4|8|A zgqZk1vH$^@L7fAqtGMT3!A!CH-XiRS?NpX*F#6YA<+%3BTm_=)@SR58cT5L&$p!5u zV(u#jm_~n3e*gAQEHR_Kta<`If}cN};UQe?0iG45WBG!|ZKY zL&N!ENBF{IonwYlL<8(DH8u4MVm56hZwVzJ6gYT8dOkCCby1^6;r=K% z#Oc7w3b9dG*~34XI}|+hIr|=6A7 zN#@5Z)I|6m3^Ndl#>xfXS_nVs&W4ma-3Pj9)ia*A_ zE0W5s-;|Thw7GE?gaqehl?Aeg-XCX%(G?>p@le=|K343$+Mdym&-Aw5yutGK6a{19 z{98pj#R0&i1@qN1h)ruW*1qyu_I`H@!X$Iunj*r*RnJktiV^j_IQbS__2Lzg26W1P zx;FG)Pa-!>`zo+kb!E61e^K}MjSY5Jxx3H%Kkz_D7#tmU;#swl0ZR@i;U=C0w-EF? z+61c@&it7v5Z2_f@qTDEdPqGK1&6wQLH!N54z;AO(Gr=E!we{~w##(+UF-!5t0Q0# z`J$7zkv$g*T*-Yr8`N4ue@g9k%uuL7LDj9h=&1RJ#fPgGrw5kzh6*(9pNi-giq|OW z;U&QCf|L95<;%EbjNbR0?0nsp*>^x8du&i52z?-yF}y!c%*{ch)Zo--3KX}Axs0AX zj{{}9a(dZNd`4|$;IbbA&pKm#%4$4$a^(N^z1-b@J9k;o6U_b&Z_EzfV+}gOlxfJnNQq`x?YlV18lu0i34kwad!N0_SdPte7M+@P0FRu~@e> zb_cNyTplPg?g}~1QC`cC{wQL1lXPA}b%jn=S?gDG)-)FfhdsDG#xIz+xTh@~?4m6fXLC7kMh>F#g5wzDd?aAvHv34&fVq9}4@U zGUd31l@Ku1G*5Oqa)n4rW$8zDZ)Dm37MM-nX(5l4wAKIXM;7NLLK8B)g}8S_(1END zxfF5V%{+^wYsbIHz4H%|@FEq4RH1(wE}Z@!G-uXt$@%lfKgDm86eLASlJ#nT`H|7< zp&@k_8SmZ24vZRhX=S@#bfMAhlSp;?8L3VWZwZu^(ot~Yladx3O+^4x&zwybu!X6H zP&|9~3~V=Wi)Es8(vua>+CcOgHIX5lKuXYnb`qq_26wWx?H-J0jSCJAp0X9%1jVjq zZ?Hjq;Dj*Mu4e0%n*jKHsb}hwr%JmdAc!0!-QHaw;^ny25mV$#_1F3s@Wjstl53>i z+(Jq#C3!LWU%*s(C563D9w751!j$XF<9Q$k9>uf7@R)bX1fpY+@h){nQ1DjkBDnOc z?lhp>xub;h+0vizF_=cBSqCx4lg!7OJZ{@Y%*^?=-^y6ih~4V)_SL)Ff^oiPNJ<{r zj~2x2pRR)?iaz-K=C-u7^iTmf$y_=`qd-;37qztb1)>wZumS5>Y`+4;1(9;!drny_ zaK1Rh>E+DKB|%bxZ~Bk@do))3TO+(M0lPIA%ZR~mSZ>nfyxe;?3fGaCM*ax60B}sr zd|*IxBCUf#Kc6zc_}(sD-Y_VY-D6~cD%#=DEvX6wgyW4P^qEN zMY@nFWWKf-1rh)qpPer-|R~i8u?Jz>t z_3_f_YAZGHM@Xoy<@$=E@XebyKr6}x(aVP&^56F0a20>Kf_VY7p{LJdnYpL`IjJY3 za8*^+I3@^w`fP(+8{gBS{kO2_SX%7-jeOU$Xn~;FL zQ%o|onF7KmPbHy9BS&FjVS!hY-(r8MN9@&4S_uK0(e6}{gM-ztd5ZhsfS~r>!y;FT z`Za4`C_boXmoDmpo3_p7byp>tCMNe!`ETh-SZq@`L zA(s)UC?5w0hfEgKiLCw5*gZaJ^=n2HCQvhR>3cuA+6XA^o<$C&iMd6OmKlq!RW^IU z6V&3EuwGTo0BH*supK9C34{f1Clvbn`Z5K+j#5Fq1xBh@q^-R5Cu5Cf$%NGqg(n}i z;Yh}(gLLErVe%2MFQ5gH5;uj2O`AnCH>4kPE1AzqwVqY#c|5HN3sG{dL57s(hJxGU^N&PHm0N)9&JwQh@G9DhK_!;HYiv= z`?XQr^d~dLzdH)A#m9g&fYdE(`5q26W{vClk$qCUq~8@GtERZu(PK~ge#c4~k1&x= z=j}bPyejc5Z|)h@JLmBp0P~K1w$K`=l23#VjxabT^<-Pyx@@M6?oQe!0!{|_wh*s_ z705|276ojd;4jw=A^>7ag@bz!=!M0d4p2lm| zD~y5$6JUow*4Ni}cDN5rn>>j$K>=4LibnFiDmzf)?X$DST<3(yy8;`3HM*4(^-xok zaoA?+`}e1DW&#|_Qc|yN6#qnxN^Ib}AB}a%AQT$Y7i~35@Hsw$a6XtS0wl@#fY5wq z>F}8c7;zHs@^Fr#{JpU{ZxE`3(P?k<_Q`-Lz7_|qX@=F$vDW?j1Z_qOKo`%W=jzTN zy+drMVWDNuN)+`52L|etJi+@$Nangd6Y7)5Wi0A2BOF6s3IiNiA3)j{Tp(Pxu5||{ zqhM2h;4~B*tF|>$N}u(y{laHhQ3>b-YAE;btou#!__$hFB4~+wQfcOVP&e<}mnL=$ zqDt;6aHqSNG!Koy_Nd?w0x8C(hemAF?5*EeZNuwFPyhk~@RGd0fFo&(1WxXKI>;Ug zIi6J5O;+{($_9d49d50WB@Xg6^&W7x`?QIKDQ&V3pUlsyEYm#n76xGfxT4DPy1HYK zhsfb;iyi@ak=Rd(i+kfM7zw)s$neN`EpUbI4HBkkdk=mY;1kcN+!En^pUVBed(F=Tqw^m z(@aDUK1Yl*->GRh!jMBoWU$EU8p{L7O7K;U8@G{w- zLSn&?9JJY&?tarwJju`h%d$H0f8Bnl-jMxI^odkWycPu6xBtljkr$E`q!WGp54`w) z{U+x^PnlOY{)tgd08ROu^8Mf7&i|Q?0t^{~{Z!J06sX9ME4Q6F9p>nhAE1dA9DiLe z0#p>Ez4POpAGpBvb?$)Q6=~K`$144?S_5CrR9q@tgsv!<|2tjBA9eXzA_dFIh!#*zlWL*z9 zF$a-D_)aJBiNwp5epBC5`J^cjT0>0of*J75$N?5)$jXFSb~DMLOw6e{&NWT~3K^FK z9L#cejO@idkQ!;Os@@X!KB)kqye$Z;Z3n5AsgP=UFc*Znn-kg41d_or;U|dgxu)s1 zwjV&-@zwyE1vH2I+2L9WmD1wpBY@}V&@lemvd^8qo-aA*D+tWPT%T^+zY zU;I%_fpgv3Bt}I;O9E}fz$cEPgThT**!R(g~r&@TaK1kNk{Xv18B!?FiUJ$yir2#H0vzli`MS_U5=M{vGhy^gqq zo3{9(3d;szYA{dq1NEWNI+P_l2JS|&0q~zjz&sGsMFiOjQOU%5lr9fF_H#w)D)CEU z5gmcy0+D~o@cn&FYNx%$4hIJZ5^j@t>R8eW9*|!sy_JRZMMFZcEAlb`o)Q8?i&YEO z9-^OiB*<;j2r7ZTNxYWMtAoZs2m+ml0U!fX{61b$?yDJi;(>7t^qz$0VK>mt=!IJO zk?S#QKsQ&MzqyS9Sd?bH^D;78Xfc8k#LxoJByw~hrpXfO!aAGMk!mDt{LJURfqn--`7A9Osyr}Ln_^f`j@MQbta{TeAYv> z@Z0O}1iwr|K0J`#F1VFl2NJ3J0VLxQxfGxe>&3{d0f{BxNyZES6PAM{2_(7kg~E80 z$3UwC^JFz?Jb8WdcEB~r(Exw~jOs=rMyP};#47IM}~htu0`9PITRx#eKbD$P;Y zbb<1U5Au3GoGstlV$p-Yd4BL!vM&{Hg=BEyVj-A@K6?m0?&!FIb?fTn2ab_ZE)cZN zpo=}jdp}eY@fIv6z#F zBOg9|s7?r$tj&?m@z(9zhP8H~GC=x|ARcS)lb%Y5i#soM-(VI4G2Wg<{K7y!T2CzT+jgtkrhM5Nxs#p zxm|E4nZN~fTYoFjuU2mYSZs5qUJGVW=pArsL(Hyg2ax#3#{e$2S0)QNavcX!2BIXA zCgPW3UEX89BIrO{l>6ki^AKz!U?(84)!sI{e3$*{1;{qp4d5)Vm`0h}`NV&1d`tXP zCl`tj#5iac7PMi1{@ro97V8Qq8x{-_DT&2v%f0zVUBr~LmmxG7K%qRPm|MRt4iRc} zCt0h*A7FI?oN~P4y&c&$<0YXseGqWu7?DTL>~-?{M`IvH*sbLQ5JQwbu)EMUhUV!1 zjK50yQ^){VWt=cowr7SbhceDmaZ;5)|H_MIJ+8Pv3Y#EICT64&cUNE00FM(`F1?x4 zV*U{7CGj;?#9v%pYXtOj;lA0G(p34{MdN3uX$?9_UeKlFm;s7KpZ~#>EpQQ8G!OyZ z%okT%b@SQ+Kq9h=17T?!a4O9;Mc}FMu!e%y0`=-!v}uEMQCu9sQmXfTQ2-6pPe5** zcnTRFkeGu*3HH4KTwK4yr^|)|tn`4%R_+c1KYz*$7+)`>c_=(gdiV=C{3=P1&|iFE z>#bQ3+7E#t@`L08HlI%sYLcPiy(-zyVi>+9vKiKLpyQyS;icg2;wAxCMx4S1vF+Qk zZ<5X|YievJNGT8!5#=ris?Z1v82h6D!pOj=d`HhjS5qYqRs=2#J1xl zEb3{*G{0tm=u{Cb;OA^t0QCpjPKzMafphp!l9}gA?e5c&yhn*gM@PE8w)`N;tFalw zZ=s6=Foj9S8f3FePV!!np5cCT0n6@^>l4 zTpxQPMGgg029O2-b(_+!B{hbIhezYRdvo*~02dMSfH2F~0*xPb%yu(?QG9Vs&JeyB zM#LVy`a8D)a37mFl3p+nMieJm3L+~$9>d0H;eFFYqV}eAGu~QlUOaNpZH-d&Jg@0ZHH%B+0hbJW+yEuDMQiDU0}Vx_zMz}!P(hBS!0wC zbZ0#CJs_S`@4==Ly#Riumti9z>Va(q6afSN`(C0jaPUkKuqM~7^u^CoE8!bU4&@7t9Ccy@M z1#&jKTH9}`&qIc+r$7KMV9S(w_ymCEzL@*LY-3-)s^ywAdy`$`z1p0>YtgW`cwdYT zXbh&}=6Gp}jcK7qDf?&C6ao7H4?4!ZeKxk&*GIYYp36vqwYaDW0+kUqTn$t;A-V#k z!uOYDm59c_vP@wx<4$@W{=|Eho*e0g)9TV5f(L>CDCz+gLkWGJ9*B}zg@7J0gllTF_+A1yF!Ryrc48s$vLh`ZTxe0I z+bFclA<$iS=C&3VVnEg_GC85LOs5;i_y}fnyJvC6JP$@>m22~a@vnt=(&F2+_UpFJ zg{@|Y!r06Y)C1O(en3UBC{`{!U0B|zAQZg33>|TK9F2Y%H0UCLQzJ;>i33ne zb5NBT2yZ)Njo*y&0hi)uR>Rs78!QTN69Tb>WEJHXFDfV-6gz%&MbQNVQO8Q1gx>j% z%%2KRs^j@%0nQH(p51aioxi?f76f*o4y+RZx;{t(|J!W;DnL={jb>2unRK-yJ7&y7 z3^!SHFKa^Fud6}%*sLJ$!wM^$^~)t#urn+%fNx{*@nLUkK-Ue{v~){FBrdo-yU99_v;=W&Jsx7xdkK* z6iP-)M&Z^r@F-`pB2Xh|O$=iC=kjDSJ}^=2f0O+IoDC3Ral64R?k3oJmO`b0NPqmi z{-b#pc8F|1BM{Ny-!OP00XT~Y4(K`(}rQisu`%5^P?@KMgoV^oP;D1#EtHY zTI(qgec?7^HlX0AZVGVj+wL9!x3vZE_B6f%9>DnM{bM$itsf67D|BP%TEVnl(LZySGMHxiUG6Vnad~KuTqUY&ro7zbQjzAN0J5 zfhwBW36~N;cOZi0%6uO324x)JYl#X;GtC1yE$M<*%-&&VGR_fR3Jz%i_>4QL341(Q zZYp^?#N8?g38%%NVpMsMM8jQ7>=WT70q6xS;c}upXr`WW2DLj-hklS`2puiZ0MF=E zv+5=j&osD|gB;V0ll_K%Ug{P3o+r;LvJjv4_Vee@z-xz6K~`E+q#8qWt%+!-a)F{W zQ&du1(1sdxKevYJ`vUc{-1Yp9E?d+!IgqX24-XI9iB31lTXt`!03;h80ubAW`B}~< z6`3qp$rYdmus`hZdwm6bpLUD+NA%e|K>|fkU6S zNOB;W6z|jO?p5s%8P|?B8^c-@6B$(HCk4<`Wha?eQXFS%$OY)?2Nt%(ufzyW+{dR1 zB3+uhOFaZ9Is_j$o!3Sfm*Pk0iKWUi1CWKFR@=oVYSto3y32{r;ZgyalzG;n7}i)C z&s^SO%`|!th|M+o7^XeFJ%MpvbSDKpW)+;(^?+}3Ivybk!h7Wta%jCC)w)u(7?q!7 z<4IIcyHJ0c5Tf2Ko4aYo#ILQ-exdtZ45WuPE(O+=&R&Ysv_Na_+69IX>O(c(_s@LxRX?R**1FfAy&&{_UN6JQu(bdSWTzQGB)=I!MMi9HKOnx67rV0cMXnsLW986N=TmasU5~%Kw;LE2 z)!~&aEo~7o&-tLL^!%SZ?JwNY9*<(heeOAGvP#K{BxY=mgEMzh1pob=pSOIvhSj*s zX5N&k9!nsI&5%S#Y==iv=zN#=b@{*)5FG%j ziN6y7hl|4$XDJ2nVfe>nL@^DD=;KD8?*XD?MYAH_22?PR-Yp=J&(GK0R;a;h2y1?F zP6gMijff57q7t`an0D?4y@bv;5xv;2(47yfCB{ZIvW9Paxyoi#k*v$^&qL*;(a70& zjOp(f=oIM#(NAZ2*|>Wq3!Crpz-w5y(mcZ7AFg0Nb=oVuj{|(uQ0n5&GoxB0M?c~0xBb-YfBgq!v zWarXg5r#pd8zuYSK+YGrBH$XFUA+N+Rb(q^kP|ls6r_^(52M>e`JXR!p@90I@A}9@Xy@m|R$o59$t$))xrRwF=YH{{QKugg|$ z{PV2<;PWI9uZ`p5*0!z@YyRXBTrymmqZ+7-A0!D@0K6=gGz336UK+(d zsVp@+2E_}xD9|bfh@T_~4yCvRh6Pa5uD#5s|LpWU0ye9VGue|1>u{vA6-wB^CnUUM z=>=fUNo>u3Eb;FfXd;4I-q-kAiqZw8g8tiP3(o&ob$;>zRoDo8q1>|6aSa<(ANLq- z3!x&q0Py|pnY_$|`;49Fv%GglWt+kP01 zYyQX1=L~OJfJT#D_XIG!f8VwUh3Yu{VfexxGq~aJDq$@D*%%y$=Iqsu$>PD+4QQO1 zeU73YwJ5xCoF-v12RjljdA%8ac79qj&NOW8B(#w%D4vd-(eIAb*v`4+T9*wOpe2XT zemJ;ozxym)4MliQt?~51o2@7t#6iV`X)*UT{EAbNQiMG4#h=H~21v7H7+l5rSjh9}eK6yW0(S z8{D{uGo{nxS&b_vOZ0v12iu=cVdRq#Bp+>NJA8A)sxZ5@9bw=?E;h1Kh&1M6c)$o8 zyW+m0XHJnnN}`r1;0#VeLPCjbI#tk1G+Pj<2*XFf^oFc{bPDoWjhj?t(gp2IVOE&>ZSnf~wiRmGa+uD6 zn62jlPd+;Pm4yqn8pKrJKL0t)BH1!d^BEO}mu?RQMxSn?@#W6XAIgW)a=sB97 zA;Lg>D2~}YdiCdUT^UBQQ%Bg!1jZIEw*Bz-9Mu~iW8<`NqCQ&eu<_a_D}fk0q1yc4 zj<&WWPjd&~bK37wC`Zvm5hvmZ^&s-iwS!Snf3j>-#%GqkCenf$JUKs-|2|o8FX z&>fyo*--}1M-Hi0lp0RYedMv*oao(s zocct^;&sK|zDlHw93sn!2~zaDX`eX1Lcxo+^lB67$)wo|s;%Zd*~$b!Td#eR%(n%V z8aw?PBYWee-I8jbwX)@c2Ghb9_p`FvP5I^e+B}aq&^+oLO@%a`PF5v3ZSRvkhl3!% zta_*%L$41^r+r=v{;j*WDZp4WU$soME0&S3wSA|_t0BAN9u^yWI-lRZ@)D%k)t)_u zZIEh&ka2T<9xG8QWIBRkIs}uLdj{CO|I4u=Vl#q-$EjbT_Ci@uy~n47*{?v#2x`b7 zd|$qN0mTSV74~Guhi1ZjsJi^$j-dra%iTs6+9smzjKa0@$3IH}$TP1Y%C6fF+5s@@ z!{<|k>g0o*mwCgX1#DU>Ioq?3q;{neSotna%*8BTKaWoLy;PAk4)I8rc_amAajSfI z{81}Edt&)y@8L*JvX@;BOrAze=`xF zu4qp;+QAecH0nNG8`uF8KJHDZ4sDqVXz|^}7-JLUG!b^XeR4*=djGKuqAc}{jBpJ= z!SQla8eCjlLc##`m)BR9b@Va!Yz!h3(Ef-uNqlvLpdG1*n|5BLw>LfW7l$dr(`cLi z58YNli2QFO+Z1Kd3TECdPo-sbf}DO7eV(s>sbPVuTdb?<-rgp$7oziPztlzlgG2*! zd(;Sl7^GMtKpr9n)f8B(f3YADXEtbac~)Nou>wYzrvXP9S^rAoYkgDg)2nT;W4K|1 zp8#kQa`Zi5Msr&8yZkj4UQ%JwM4#^wo`1OUttEWE#&$JcuhzV)TT-Y$?Oa7c0R%2| zBjdD<(ViMFH>;$gCWY~xJPEH)-;YoBbmUeZ`l#eRUSpHDJ3HJi)M8E)YXvSyCBB^cpXrHghQD&x-F&{F*KC$wmL?;$pl|dEJP)9z;yPlCo08#=uT=BcYBZB3mL_ z*RNH##F}ev;qdTP+z2~cbubKk`3#S*EPGMeCUDB}C{<=Mh;3f(c1D_L6@6UgG{MCb3=rK^^Q z5^lYLpJ|a2j#gPhvKAk&kTn)*=S|8Y!dBs+etAV8Cxc-77NVJaM6TbG__W|%kNBzn z&#mjr9moK9>@tZnCEu0g+0@HA4Fvl6UYDz7pTgCfAD74M)>J>4*#nLHI zgfanTj8diCH~%UA0DKz|0<29*d$qYDiba7oYq9J6Ze3oCds>zEDTmum zvw^JA_FRr4+E(*IO!;?Wp{DdD4i5f9pKcuXs){F-n5wEwr|YbnLs{un*bi8^;yk{X zBt5&j2x1bmo>Go0N}BU+0?%c?2kUJ6=)LQqbuRztxOq=nH!GT@BI4J7=C^SS!#Ad;2B%H=u2uYEF zeXEYIfl0U}z-rtV;sH%n;x1n*@?Rg!g+&x{MP$MvzCTIAmH7Rz*U|l(0$ZmC{+fBt zf$H()mSJuJ%ljhg9Juh^lM`+H3_&yu3Qox{MX?HI9jpb4B4`8+dZo!!==vgFr$EPn z9zzjJw~g`cy9MCMR2KgAbcq-~5ee~`lI~U<$#>#+7|`PnfT+?fwTa<2CAXw^l}m-Pf1K@s?p0CV`0|HIi^KvlJUegBFg zAkqo~0+I>{2uOE#DV-7m2Sh*#5do1dDFNwJx({8_p>zrgQqn1*KEUJU3-)0t%ti1^DDx? z29DmCH!VIMJMeh5l*hyR#qpMm%8yjORt!I%&Y1gU3k3s6Fgv{>&Jj0UzkNLO+Nk0Y zYiy28`nwP!RM-(fh03uhro!z*GT`t%c}JWb&t2x3r0(&gVFCkpCRJlUVh!lej&A4x4?*7@Lrxpe#ouCx{(O%|(s zDyHg9IgFmI4d_^pmcK0W+ip4baUs*qCQnI#jO$hxSlIlEG z&~1HYi{bdD)k4oJjs@7nwDPUaE{L4gy2y~Ai}Qt28jxB&IoQz^IUYLuSoYn)LAi9N z?3Ed_;{#_k<{RTe2=ws1b{W|{h8sffd6H|h> zKNmGNQL8H>vG(q|Ta!E{gCvAWOF?7aduGH9)DLsCdJDQz?hHBgr%7A-G2Y7w@^kN} zly!XxkOb|@qW-Uq^9TEEUhhQR*F4pIOC^GS!otxQj&?bEN;RzECGI7Y9ewUxO;h@F zR;C2r81ON`2fHE@4q2r6;X1qZ#RGN@Hi&^+hUusw&Qf}veUS3{&gV;EpU+`LlX7Hy zeu<7VkKtan@M>x^fwy1g8Ioijmfn=d)HNYAAfv9TWol-c1(#82%VKOC|f8G)n zSH5z-PU#W&(F~l7cj+K>oZ6bTHP0QeJmC2E3)<|jhg_Z#O6zy9`J;=_MWe(BrfVzwpsKeZlr$fcp=?KAjk(D$PRn`CmLEH`p*HZ7npNs!bl zhf-)XSPc2aSJ&S@{q-AQc>gZPXG2JI?04M}tr9C*aeE(f z`^Q&5IkMY*sr|IQFO|-94O>RgmAlI}tu6ga6(^boRLCTAcLq8xs-AZ>sJ$8=bR}p{ z@gecBmIYBNLPXcP|KSp5zwZC`L(;>@8Yjo5n-1G^%i|eYw_e5VdEw<4?{Cywzf%fy z4obug2znPH<;uWaI$thFXFi*MmCfREjVt41NF2AaU7q5fa?w>k;Osyw>A0r5`hjhQ z`FqPuRa%HR4HIYiudk;lCF-V21)(uz79>M>nyzg*K3S)LL!oQ96!4|)@`kh$`3+u8 z2D+_gRC3^6GjOCj!Y8lV;Zo`=$8$sqXZTeGj?nF|im(-yG(y7v#nXDXFPn!!Y|EW>zPK!Sy%7AhhvY`U zq+7k4BB~4FKZlo;QV`+eIa!>B3g>Ou{)OH6rDJpMti+8>H3%tl-g~FsF}t3W=XOX_ zVT%`&&2B@VE%DOpaI`R)JiDNvCtvppAS{cvQ%cQ;r6s6u#9>+h^kpPBqa8%VLqgmn2t*uyxX<49Fc>C1J z6FMC@oW7z6n(2!Bc$qO`GP*0BEX+e$Is8G>+_~rd9bMZHou$en2Bjx2L9|)^$N*F3 zbv7)BmUSW5=vsSI?HJZpyUXgkU!1YiwjLyb+5{y6kM0P|i}G2TJ@`3b*X4tqd2g~A zPW3tyCvtVmRtLQFaus70O`P^;T0`s?69ohUGqQSQ9nN1CHdlOLPaMVcAY?gYo#@8F zeWS}S(;M&=R@NLhaZRvs4b-sZ>{GZtco2~B`e992Ac@{�W*NJiVcf z#-Wt-Oqw5c3-rH*iWH#Boq=+?VM%xY^AE)l9fT=QzKU@_h`l>V$v^bu{R@tc7ig&% z_r-r8rcPIfvTq97j9+f*dp_3@7-k0B!`1rJZ3T;Pv0TIr<-EX4rtO;22w~xR{M+aK<`@-jql5QETLwd1*@#Dwo>N)^5IL+LdMxJP>W4 z@y90NX41VQWEKyn+4)*>57f#mrt07J2Unhl(D#M5eDdV+v;BNacy$Y`F?($}YmJ)P^U4^s(Sk<~4IH$U)p;}WHQPjuHT*)(Cg zoxRBFcn7NYNZ=(cE| zqs#Ww*(SH|N|9?56B4`m`k(Zp3riT3w#8Tj+dPd)B4fWVU|j4yJsCxh7jMsbB*kpu zonB$7c3qEK<<3)JRuyY?$MGR$@OXMD+v_Wn)n4C(GWSz3jiHqI**4v_TJQV^uVSbQq6`;#2V!MlRl6FZsN4p7szK!L z)AEOiTtTF~Y=Qs}{_GFKI{7Zb+arG8!qe0qjBw>|W6eQk$YV!GU&E-RUoQaE9emdk zd`oiP(LxWB1~vn-V2PouE$PV=Gt2vt;)A+{5x~Og)vv1UNpRV{U!+$dt3>axT5K9l z@}MuJgacAwM!gom#hw1P5FKU=hDu$S6wXM>0mx8QjOD||d9(Z3yiII2_LlE9b<|r7 z$0v_DxU8d%9tKKZa65QSs5QO*g+Sad=+Z(j9!H&9p>8ntndPyELU&YV#S znl9Xq&l>xfIHAyE&~lrsxplnEsQX@7#XFy|$Rvy2ws#5(NgQvP-My{7TP%{L9$gH$ zshD*=i)-F9LuVApBtZ+sM`DlKkWshl&tbV?XKJ5a%q=9F*#~nRMVi;3lAH5ufZo67 znUcA(Mw{#!!xB&WK-?~{n?`CNhB-#(NSbm^y(`1+`;Iv;2vwd_ebs$xux7R!-#OmX z5N1NpstXUwc%4<;lD}3weu^QV_#kceN_D^p5u3v=d?$$hVrmV>(V? zV)2h<<`o^xi7xiMbW?LR}FL;<*nrudD%-4?wMh2b!Cm@S?eZz>x$~0Kr*T$ zEf3)JCnP4q5w&!_ncCp`6je5sfK%{u#FMe2ZHCr+V#e?@b@EK5Z-s6s1wK5Rjac}Y z9oh{s-!xAKo4Z77K!$CD8%8{poBBN$f9p}-xQ{9n%|=Fp+w)V;jx`V?%v4cm>f9Vd z6lNJrDj%Z5d#mk6eA=GdGAzjm_%SQ1t_db%=&T60jg{9$ zqdNW!un9|pH4=EIdwR~7OKc``S02>7)A+)@Q5)HNnfZSX0iC^4P#iB`Y|AbAsIaRX z8P~WMdTd^#y>?oOp7(hTc*_KFuL~|Ia&4i7XuFS3jzn!Y=C%qDK98UTXuas)n#IZ(?yn=ykS_1;1~8 zuICgG5YQV_F%Q~fQEeNlbDg6iQ%+w0oGKwlpkR5M#&r(~{`NIZF`S_r%$Xr8p7;%t znZNVTx9D8>*ZioW)zM#3ZCKhjA!;044i?S5pWymR9sYQFh3T~%gCh;Q%VQ6L4*kfN1A%?VdT$6p!7Ue)u%wbW z2sK6OW6iX~#imq{YpUy-SzL=J=ez;JVt8Y@4ef?dy<1IaMTN{bGTU?r5Vg*2XM%{k zt!npwL}`^k+Le4#<*VYz5M^dY-b#z50X|hq?nfV^kk1@BFTdg4T7x2-&L>R^JyOEq zK~rN`dSy)S#hcAy>rxZq`>ACnnuptJ?Cato$W_)cPOK8W;%}183g8wtO>9X2X^EZnm0Fi zoP5%4ZEiZ4?_NoFl~MP@rB{2{8hGQF%{F0qq9TteC8D{_v{g;?Fa_bA8u_*iijUo7 zhF($FeExiEcSXik<~7Bq!MZ>!B`KH1YVIthO$+Vn=lhF`{T7 zIVBKt0ko!ZaPWAgF!#Oxob=4twOjWXHrBsPi5A;7 z_ar(XK*tJJeZ!!-9Lcj!7RhV#R-O%sxThbFdu{Q$GkP1xqJn2K^0i=yT4nIv ztn-M3zK`g+P%BtUG`_44llaoH;Dc>2y=+N%@jW;7!{r*WJ#vpjC#r{!*w~!a3xuOh zSLeQdO`;HphWdeRwXvNgp?EIloaIpeWiuCHowuV#y*l~2?muyoZ9i8Q-Jyjn(~bJW z?J1A;XV+!pgzLDMk7j}@)X(#W_^*43wi)zya#>B3^O{Q<+o;AD?RO<}4`gh;=MJ$m z;iBjBxZc9nk4VNcWG0F@nzo1Fsi1jSIsSQsy?%H=0Anm2{~Bkb^#kz;5kX}r+ji_j z#J+hGA^2Zk@lyWK((-yPK>1K{Q#Uqj4)6ENWo$?A*jVvBF*gthCF5Xcr`JHufm(>u zem2hnPf1^<%U+j(;hkChBC_|FzrO9)4#yOgxcn?cME4?Y@c%|<1l({yeT+`U;ZF#! zvWP5@_4HiJRqy11tD9>kd)T|+Aahg|n7 z+0(^7`L;Z7`E+TGR+?xQYm<-i<2&tC`H=tjl`FWLt2yubb)5?fqb#PVWBS z=jgJ6se{!t{fH%GK;)yz)iQh257;DWZmEIFD|5BKZ!K)$EPp=pU3n+ncP)x={@eKI zwzVd*=JOI#xkur*d2ykXJ)a3hbaEAs3M&xzp8weF5^yf>H%CX`eZyC}{^IO>xWF?9 zAOyK)`Jmo{tRc5o-v6>?bqthDC}hJ2D&*Oo%tFwWPO-xs4b zU?uxL8(MaoEeA?tO#PH=EY)UDj|+{x^Vzkz82MbC+!B}XuY8QXXQY%<_ydm4C20j5 zZh{ydBH*0lKD1xhAS~(mWC5+{%60WbftZuk(m<^O2QBm=-zW9T8Jn9b{d`|)8a&@U zni%QQ4q!Kqj$)2?KS1!U3Xb`>A7m$S?|idKYsQHglZ~nD7CbI7#^l_qZhS$kQJ$H8} zHNM4!?fZ3oJ({nSr^L`h=tkgtZ?YP~%X0d?=lseLBcOniZm3*PO!LJisg7oj!3YQQ zOno;?^<7_IJqPxkGWTN=A;G&*Y{{AG+1EOJXi4Mguns)Y99M`BHk%`UK>n{)5YC%Y zrh9qX<**s3>46nV0cgxUD!y%k8HMQDporV}Ha?K?j{7n!%>PXSarH?Xc@=(3oQWH! z*`{Q-Zk>1WbrJ>1g1lQJ2J?pXpX+49Z&^xwDyf=!&Pk$qnqRT4yUK4f-4qKh^yyl~ z@Pjp9yu#q??kS|>CS!MEIb3o(iCGNATzK`?KQ9$KX6O}Rsh{1VAA7ho5H7SjJL}_t zO}sN~!@E{3kVmAqndp4+x;D~MGf(*9P}a1c+*k=yx(YWR{p8(?lc>f8mI6_ zP~34N_O;=`dd>3Kd2ld{^;(FL4TIyh3tt_5^;$FGRy4xZ*@x3#KgjBt2gSj&J%Eb1`#ppb`^MF zv8%(mms)7ci-qmSgGuX-j=u+F&lsuWJ_(bfyQ9MObR@6(rpSHK3=8q6_lHm>mOn(j z_9?ez<`MTZ1Fs8gsqA6KvEC7lQz>O&rbt)6)YYtI-%|-*Yzt9IvBJN?B`|VXh%voO z76aGeMXSy=J{xeUU>*X-Q4~GV0RT*O#qX%8>xy4)FI-5>Pd&rUtVkYSZWBbyVQYAf z@xH#ZpCc}ry4WtuK$*1=IrsBUlTpOV2#YrwC>rgCbOc<|fwG=xfd4XtV@636Jj&G< zIL8ADvBlGLGevURcNkJ%j``8Q6cX!R&logGmGD(p7my1UI~2%ocH)1sQG8SFH zI#F?pVg8$8hRuYo@%Dsma_v>Q1;W62+l59d6 zGmN9*ZV36JfGhIOgV~YFxob#aH=0rPfZ`v7mxTn6AGXz~Rc%ygOLE z+oGd8`WrG!Q@SC=FW*IpDC$A#TR7oA4&V}Q#r!=x|8^5EGw zMT_$qI1zgr8@FAZj_qfyF5g-V((f0n8rH!vm4DoZfSsZcn|}#;dh~rqs9M5T@|AX% z&@jOTqrf7eaO;3C3lt+*wk;dFWof%!ZkbUHk5Y6^H6>ryv+53MlV9%FsCXhUNLR(f z+cuXmxJ+rl{jc%y_1*K~GqnhFrI}-OUR!i~EbB0C@p81wo6h5D0^kN$?8<;QK57ew zvxp(c=DyOQZk&Mg!Sfw8M!ZnPgA_q`399zzk{AdN=O&dZt_;OiNrlpqG4T4K~Ygq&`*>%KVxi?LSn z^5wJZ+Vw|~7!>^{CaooJmEx<_(wp$<^8D$$5Shv$#(N8Gi2YTB-_)XER4!^}yp?e1 z<;>mLYJVG+yFM6=G*{vbG7lbdk2EtH5c0}V!kS0oI_n;;1>5x?!k_K<*A^SmNClZ| z@O0itmdtcxc$%*$`_~=P63EY(s5lI!(8HNon4Nulfzx&W)wQ4My~cqhZ4(GK{l&m3 z+XjlY3kCv=(wPQjIgBqYwIzq{-=C^~FwZbCm`UrWQKrnyzcn8dHUH|t+WVM-tsr8q zHnRrz{&im=r&ZJQE5okrO(P=sCox7nX~K7y*yIQ^kC!r^MK2Zx2g>qhy6p}>lv@F4 zDEe1tus>P)8M`ZMd@Rwg^g9~Zv9Tz8^mhsG%RTuvG+N$;dPX?IZ`ZD(OCKyp5dBoO z*H0XoR<>?PBv2Mu^E^j^@3Q|ThCN-Xv^YZLzkR|P-h8?$H!U|djEgax7W`4nUHr!4 zf{ENs?<@=r*F8?x)J;tsbo;5pJ)>D@9cPYy-Y5$g6gN7BahxB;|N&6(7jz6HlK% zNOR@ZVmvTG=)GguL|iiMA1X7ecsKR-7q{t?Mhu6{xbLVT8}_JyTKm40qYDyi%0NX(3%-{cw9OZoJAwKdhFO{3PM?Brg3g2O09C{+Y?y zgGqdYE(bf~JhsyHVZ^7JS@=Jtt~vh1we?hO^rv}9MbgMsA@!YsJe`csVcf{W4ev_* zmUIEpjilOfjGpvEngaP5q-wTWdQMlaZ^|p!vReZmMDlQ=pRavKc1A0_-3`t;3-cxG z&zZ4bdQM8t#?z8R!ng*Ow<(rZhTp}LlObHM_Gvk@R(131BEvA1ptf+yV(C{qB2O63 zxIy`x*w0B7{Y2s*JLk~n9#GN5j}zYwS$+`$Ypb8ay|SvFWVVmU$UwJ&cd#7%FOl!< z)W}zRAE=!<46f}V(R%`EhugSjm?AT`fpcNa>eDa2cx{ho*O1vyNxa>}Nc=|72x`4E znR-O)y9{$WNjcCuiTr&1%E^_~m9aeKIB~>S;DIR6yu_CiKNhoKPFfeIx(Zz8SDLx~ z#6~cjDDgK!p@k@hKVOg)Goa~8Lue?;`MY%^R}s6*<}hI%3x#|JPdVjk*Ul>GPaM5p zBZ)at6!n~a*N24T{o<{D8O<`e1Y%aTG)X5+vy&rX;t7n$p%jPO7vHOtSg0^}dj;ZM zRjmg%ZL8+lZe}4{^u}{CjQZe^1`uhq*}Ykegmt2O3k+;DJim1Zb(P#I*F4^on9E7c z;2o}aThlPTRX#n>){yrwpk|O&yALy7g30rJ^FhRsi>@3@lt*qsHqFKaP=74!KI_Mn zqJT%avoW&fSaBnF`}{6n^7nv6#zgl&SgmL^gT%a>x@t~5 zQO|DNG_mEHq#2Qj8nJ%7C}tV=?5l^vdbaarr77e*G#C{uz7-Pv2nvbcVTfE0%Er+6 zS)DuCZ}4pfHva?p?ox~7=Rq28!RLAfTaECOU8^~}c`z(8p#|0MZW(+>AoZwXa&ZtF zA9CQ+ESuk_(_f*bF_P0*8ZS%wzQE(X-T1WP(XpsogL=$RUAwXIvE44{02*IEj`uD| zk40k^dlq-Zn5uDpRB~&$(X^Lz<7V1c>CLt_g^QOShH8rOxgYNetyS32K1>f{*5amM zPwR@~>Yg)|vTFqsl(j3BC~~S#bxh0Cmn1;)LaO7oZ7`z!7579%1xMRHm?|sv;biw5 zzu`u@TC%!~$9PmHn8Q{AIM$vWyDP~Zb5kD9{VXI61ZvG2ZH~^`ne4|?r&7Y=dB@gL z!Apz%p~ap>YG0SD55xNX#=`3l3;4N{wyXK%gtT@I5TRzYx)A3zmN9(f3R;%8a$LcLi0qs=r*g)d$931&;KH<=?Xu51J zAbRn4);wKO1(>%U}zF-&$RTRY^pm)9$Q{(=k@Q?UmzDkC-~ouG5e+(HaHI40njG8zf@ zA`xXFt#4)kDYN`-BiXhut5)s$B zBxfi^x*KxO9G>Ek3LOp-h~WJ=h)FdC8RN}N*om#Oe%+9KX4$rGEI#M^U!p`iCx3o4 zlx{&hFhZ9y2qkooZ)gLE z0K-ZJ_J-+Jw5tiukZV4qI?YszfYs^ z#b~&Dt4Mx zz}fq+J0p%sg6mXX0H$VB>*CaZeE|y(Z}is?{8G&2`r+xJ_J8K@-qPO(ipe`)WmIW~ z3EFS34fP2{oJ8*B{|dguU_W|8#?Mk1h4ptV|9%^EC&e?rr6icd{@JO6sMkdF`!!*} zqA;;qM8a%oIsZG3`#)Ujf8NON^nWSTEu((^SFJ;!ud)Ir6!-p;nZhpP{P~GrbuNsk z548-T?hXF^?wvn(1P&BPkT&mHpal&~b^!$fv^7MJgUvI93I?*AE$_PMz z7{IJ3@ARg3|1xto!Z@ssDa$ zqQYJ}U#l=lxS16Yxx(LGk}h*5oXPcdLD9f^?daL+cJ4wI?9ki?Ff)mYqE4%0kfVs_ zb|)J#TJQyOrFYfD^|Ai^n%ctF``Lp_;<6=;owJuqECQ5amK)?_pa6QrcYqz|=8K&& z$Z&m(9bkySFV5(dwPH|7f)+}9ppdO_V%n4bd3Ju+d__z-@dQETRz#{XuLo4cWkX@@ z8}I8v?nl~2cc$Soya8X4K*4Jbys|P@oGwh;uT%B-0oZc~y-a{`0MPjHlDmRG@t?6x ztkY!hrlVG-%$z|&IcpFe+&n7+#4Vu)HRFwsUn!SsMfKL)N z0mZTmN^z2sl9wp>k$`Rlgp3u8GVoqd2czE+80r7)L~>|)OJNl9~oigYMUJ(Qtx z+kwTJ&EE)M5Bca@DBQCLa2)cV=pHH!q$Y5e(K2}hLyK|vB;yeG?D;K1$VnAc}s z2D%5-&5p}HbgTz@IG~9KqR(5}0ljGWLjsh4YC;Q@hqv-5xD)J&M4s(fLV2rH;N?c? zY=fFTK*P|JD13~(f%ohO(A-+YF>uH-%x?iJ1FE8i>8M$K{wx&>Sh&=3{*mq3JqSny zP9nYH9 zVCU=eF=Y;zMH&TTWM+8Fy8!(K?%EY}sPI?I*X1&!hALfB$=Bxt2gQ^7n6&$V0D|KE zNiwNtrHuSS7G%KZIbr(pJq91{C#U&G%YNScV}>QWF=|Uf$pUyiKTb`Xys>`Zlmdh) zII`3$nne$?fQ$|LlFpvHAUZ@{0mnRTm|A|%UqK-)lnvu+K`#ctD#{n+36Bu$jRB+& zXym!1S8gaLh{gimG21_pznD+;$Ja)$oK1iT_B8?Ip<@o#D4-md;ja{ouhmION!qj; z+GL@7W?iqB06g&$E4>`RP4eroA3q>Uw*b~Sx5vqIE>qwrZ5b^94n!SJqW)}V`K4gg z!suE$XVnh{S?*jhNBaZ-J0GU68G|p`8jkLg7k~fy)-7`w_Y=&((dOoKs90x0n{KSd z{{S_&ij!4OFvR^a%~qit-Cw`buF-@@osmsBh+~sA0GLES0FxqZ2X9n!!ov0IV|;_R zm03i4*8dEgb{_zMG=9PZQC9tEkklLNS>^Zy$>OodaZNtpdBA{a7{f4@cVu8PnQ1e$ zBBV5!!hiWbXfM=yu)?-m9pV9u#F55 z+YKO6U+26BFg^kMq+3qW29lq{qDMbX5{nCKdTV#rN81W}?n4ZlprAg3v2a!`&;d9Y zm9a52=Nkb@N6?Zpy&p`u_a>L{V{+M$>z6})SJ079ZW1YhGLd;lQ86q>!2TVS7GUE`ynp)o6QSQ_x} zuti?ea64Gaj{^@SJ1`aoasY&9IjOW_~pmpVY81N#}Hrs zx}4gZD@XE5ZHvfiXG!2E5Q$YyX=vI(g~&LXbY|Hg9Yv2r2TWE5#W#IWeysimifn=! z5o;eI7yNT&uq@-5Ar{Z!1bd5 zpc6>vPXNCuX03LWp2gmNoM-asxyWi~{ZG!iY;1)n2m*S&8MWz8j4oZN^oK4f_8k+nCG0!ku*lKMC00=|~$Uxx+ma{&oB@E6LJH8rRK zCnV*yOQBz8xLai2O;)uuB&93>o4Q$c%qJ*ZCz0OiVlK==lb*J3LrgvYuC(^`JJ56% zzI}6o9rg$Ys*L{NT!nTzJg9bB!teVJBLO+(-aEzR5SeY!jUQXH4+f@J0VLc$2dpdg zvsnx$ABtv?>Cz!hw-%j#J&W+u%`?{dZtPLV{Q2c7=Tl$QJ|L6No6+8Cf)qbssF&Mb zC+8{dLCkO50{FvF^jUZ(o$)+{tg`+=e>}u!{OzOrN5U-17XHv10E`uOBnt9LmqGAt z5R4X3S%6P86oIciQE5MG-0wLjsvZ^vTu%TIm<04S!2bW##bs^0Oo6WNreF^*i{Me& zs9|#_S^n`{q{^+hQOxw`K)WG8j1^4-wO<#wVHcU9k|f}0y%Emn+LzN^oD%lHDge;-{oSIG$A>$B z$G5U_!=Q0+DiHCm^a<6e|*g(Diqb+Z8)(O4Gj%nJ^4PkhPGg->(Wc~?238N+w`*NER`bugfCXED|Z1x z%)(u?ZfEP^;bBv8)pZFF585lYbV@q(fj7OnX%FS`t?M7yiIll8{;* zVgy1V6<6*=P8mr2h0-BpixMCGlVmD#W#U3Yp!i5=Suxf zKfd{AUFVlhLH0KtHI{1v*hCcb^&soRi#p4x6>K&QQ-@y67DcK+w4h3t@9%`kq+arX zqG3JWJ1MTu6|3yYl9t~ncYkPW@->RM0AqambAif0aMDp)Qu3>a@-D?YUQf5XLXnUS z8V*}NpgOKUBM{fe{2v~-XFH=kI`nBO{r)U7BMlZltepWFEPMJtZ}@L~Q;E5YX}wKN z${irM|9>A1b=W^MUoQFDeMqSvdc=fRSwD-3GdU|41FlKJsyBi%`q zT+7?{;l_jXCHd}D0p13$o*Q%g`j2Pis|$J(Sws}FNVtf^{T2TB;@qB)jEiZH&MUs} z!Li;nqf|h7PbrJPSJ8mj#j`^snn3*HnZK3-KVoz90>JbG{@D)136uN@{b4W z{=pBQ=zjI=>she8M1O-wzkC10_WwU`0Doh4r{8Nc%+zBgy})k@wLfG2`LS6KRD^VA z#y-JtqT0i@z3}uQ_syekHf~8P0Ww}dnaKY8lrc##$_t@`dNMuh>$jgHqAS$gjVjnf zse}h&WG^S!-i8O~`#VbpBJqj7Zt~*PIIJydmAbp7QU+49Y~J)R?;>;qFU030 zlzi%#({y?={GzWk<|WUD#Kq&oz>kktHc12Cd*^dgMZ?TFBQHPKxT{B4^>PVaSkn8EEtaQ?I7eLUOp*m_MRP43|ivRpm49ghk%4;ex)#j|og%AJy-F^-|N zHE3(#2&lX)Di?6yFd(nP03#CS@~ql|r!W!sPmHU!(ZD4!`@0&w@j30?*T<&&dYtW^avX>5&bp*6=>p|NLL_70`yLJ24L15q-Wwde-~;VL7X4`!{Su(ndfs`2P1} zj+A(U1-*d#AWz^^mN>0{oYLz}PkNMH27=3tKcnop45~*2lGh`l3#ZdM@J>4F3Xy-W z;D-ux?-GR0-mz>>TEjV{q}YlS_JUXXO8^2#GNiYRgPJZpVBenXU_Z)LjAus;ZjJ2E z#Y0X}|F{3^uhbu}KhF)>K>sD%j5Pjwp0kv|Et%o}=k@;|7a&ZO{|9;HX9)gp)dwDm zkT4BT$ocP+{%fpIiV|TgVAd+2!z8p8qTqMvTlnXLAO7k_vNWa8Jp>?DK&B5+&P7G6 za=v{CmwZKkEZOZ@Bp74>*aO@9Aoy|{kkb3{KqniusPqLkyNNx}&PFZh@Wdq~`fDpM z_LiTfc=%8|3IIm45fV~8C27c*6pm@daK)8nQ$Lo?1&1yh)BLNCR z5nd=rcM(un$M5UAKtl($PKbebNEw5nqeBYSX0o?1?RxFe_6I1%9FR#bb@^_7>(N+| z@j>v|&!g|X>>t4A-63WD3O<9jCPY#@q)sl;Zj6)^+pD!Z0}A6T^B;W`zO^8l`w@_y zg+<+o&;PHW@lGu78o07{9K$ln*dQLl{d@tZpNo`?jNf)r@MtQXk&wzK9dsJTMpR?J zm)0pnH%xBx^XEej7BwBIc&6hXYW8`<6EP5FvOxcwn3x#h1O2cU%DJ=P^@^jwxuX|m zQ|%na&)=ZuWhrpSj}>w)e>X(5h(t}4UkeXxZ_9bO1ph1&k5cdj;a5cxN=jklCU0qO zyLAcnHGK#g!Q$pNG>Du)R#8@u#0gaTgK{!0zn|zliTGT(1t7TLU`T(G0@^BIvV1>{ z0r$TP`tCf1;Z?N&TpEfkdxfV2_-h|sx98;Uq(cY+QBUL#kRs|2*9G*0Mzil>gtRkR zE>web0)85}datBX8o{F-{dNL4HYkWKfK~wJr>mvCtcJB&-3BFQSkeC0$G}U4sYZxS z=MEfIz}1>?^?!q?eawCwt}P$0ael=gb3{rC+;^W2P)@d>vs(j|fKy(}KDaIT_%Gt1 z3jy=7uVb;*Wn4X+m5Cx_iJd)g*VKKaR|WDDaV&=KEP<433)2dEV*m`7S_gvRhXWaM z;K*vU%4qID(5aF_#PJ_$|9QpvPr!CnrjLgPE(X2?JEDx+>_0OL14{M~F`v zmHPR*5dUYm`U^-9vZb}IzwGuinQ}kOZ-J8MK2z{{WlFas_Bt1?{9{jh@gUyP?`gsU zf^if`V*nomUOZ(ugmO7LBl&s@OG~lPa^WL7mZT>H6lyEfMfDKTHh?4rPxt-%cZm_8 z#NE_^hyT8{)hO3HbwN2VM;V+{B!21ZTRMaT&?SYeljCKsl^V${vhEiK+BPg6vmrds z-P}YVl+)i&zpz$U1MxGXYRZ{!&(_LDi!k*I(~X&&q1#m!6Hp;pY|P+60WuBqUjrS# zr4~X@=?_7d8PN{KH$jF04&3`?(J(umP%N$~5amj!!1YbqcjL>NB7*wb*h{8dygRZ?*H8lVRml@7(t{)u#WH*w< z&^Xawp=|=`oND)h6AbUL_2h*txRC@ue=a1yt?Y^3g`XH(?y`sp^b*@?^CZ8iiWE0&EcEpTQ$x|ye zYy1?owY8y_gxZZg#kpwc|4qW4Qx3YUaViL@THyvYPSo2VuXSbgAlgI7{oE)Cj)L6q zCd6m58daJ>SfD@A7ynm~+31ql`t~_2uSb)$0hl{-1(qOdTbG6S%xt5`xmJ}+pjy+A zcS+z<311^1pl%>=c=7~lxN;PAO{@2RK+lKGP8d_+=Rf+a$eCG9B=n?%-kf3mRMtBW zOmw1!0Z^DefZt53ChfvVY!y)O`l8M=2w#S4oKX9bqjxFeK7OW;9A7r7rkl>N^gs71?-h7DYVG;9FSY3#e{s4USf33zMemcN^T@2sn?7 zd@!rU^%R5LGRqpK`{?r6%;3l`U2~#?`=aOx^;1w{%cNag!5`0fPP3}Of6Ynu>C+$3 zNlo2lDFnnN0lq_k3q#8EdP%yjjmM)G$K&?8SB0(KK-IG%bZViD1ZsDpVq4`6a==Ku&~I=wt535%}x6rkbX zmcKZ*xX$^YOTENQjtk+StIuq@2U}sbEKnN>0@eg`r6Id_{9F~2RzX6bsEDe0Uy7J! zJmWNvnVj=z?)8z>nk%GHwEmp+ih+Z0DscD83xoHCWIT`ZXf$Pd2LaTIYjFrvtRrdC zBS^mNRi&&C>;QRi%N4@VyJIeyQXV-qdDGGUfTEQDHDJT36Lyw_g8DUTu;M_?&N(4| zVV?4MJ|UopZOPx-4_aZIse)!X5psT|cvo+qiL4gDT%B^&2hrUjZjbxvY8JGNc|Fx8 z#3!wqSy53zsGFNEA+32{U;%bd^|R=S+?0Q8yaJ|+yWJ$habx#Wy_>*G4)2>WyAY-J z5#N5+0cxP#QgF~NY%5Lop&?RgNwD$+@G3vzYK4iTHizGAZh5`CkF&rG^_z#B<28so!xC3o{(GQj% zH4Wa)CR)@0nZsoLKo=g?ZTFYREGwN3b{;nGN%%Po$?%bQRKm!s2_tV0BXDmB zn1Or?`wjUWOZ+{eUxM1n-h}PSxVhsT5(EdYS2U*f)4zMb#ukSs9-XCSbs9~Hw*o7U2YJ#!nd zv{OVqAv~UKXD+NAsC9Xw!7G7C#siP}- z*FgODl&cEWHJ(DXf$dkQ{sPpn6VN>(Oq4Va0W ztRIA=g3&i$PMB@Vt}1gx8)?Bo9uy?&;ie{HksCc zrm=od*Ty-`{6jR(sZSQp*+MwLOZ@yaZ*926S=W!e$5M04ZruAz45)lzcp^vmr_(uF z=Re%9`L4WTOFkOq=XYfg-h$#)T2sJQJLtXT$X>ev>zs z2Wa$HzrpULS!l>oWqHF~Y#mN{iu}Q;6lWW5&V~J}jjT~)eX4aH?W!$7!2#H`C@IAD4`z ztFpFj1I~jwU5GS&u^kxClt%HVO28QTy_UZw>OSP>;?QD_25R!_&^{)lQJH6_=#qve zbn+Uiyg5=0hd6ORZ)wx!^R_Ocl2t>WhY?p+RRI~=6gnvo*C=DhL^^6!H0Y++b7qit zaavtExTp}`+W9740)vxA+1dQ8N*pFqS)q< zg*gV8#-=1jE^sxkK(Aca(EM@=N^6!bfcJS8UGAyVk2$I=P2PkcCm4gMqVw%>{b*6?&fmhd>D3}^M0LG=z`xjC?0#*m1} z>Zz-xriKZM^UNuMkbQCjq}-w!_-1`hpTyikpu(Gx!z=M|44LXN}4y0vBWEAO8M+nepeVeJ^rSo-E6sP>k=2lBV~SQV{K zcdm=RvBz;(?4vd~;`=(3A>}II?q*~C(L=Iy0D@`;KV|X?l+qFr_M#mTEOEyhfBX59 zoe#0RL98DRdW@2ls~zdat>5RNcB#(PL54YyRa&IzKvw}*Izv&?nm(U#{OHFbpK(lq z9PuA75DTOTOnU;XrwNk)kp{=S+L;#gCwWl>NOT<7ccToxYC1)FEIi@#s4I6AL4C_` z5!vHT3k50OsGOIu^I`(JLCC&RnvYJY7U)O9TWf+hrWkf7=03q>XuI(pBq_OH665>k znteO!R#Y0*&MM>W`>!>vX(RP?eiqYtol*g@xBTCdA8^9t^F;w z5Yw%#>>SQv^z5#a8BHhq;&%$?d|9Y?y5G3R3%bq*|HL0R#W<;cx@96O>wjXV1Xn_x z7J@WI_tx%k$cMgum?SedPwms=hDxbP$LAo7^)(cpzi{D%6uW~T%BFy^O^rk~Xolpa zdM8kVf1RAd_ZeSq##d-vdFGsIAyxcI$}nnkBaWuKM^R_{_Gzgcz`AL1s_K_ug5*P;QMSy8WwcJ9#fWDtK1Ok;b9**QmYUKs6aC z+5T9+m?nY&Vclb&c7D94+S3}RnvX?m8z8FGZ;xSd+R{2 zc)zTIbi(fjkeqc=2De`;{khN=o6-7vz1Z{K7J(WM);}F~;6#oc;M`6y5-Z~_KTGRW6sVD^A<(Nfxnrfbh6+7*pi3x%YXe>?>DG`|Mx%Qm!jfB)Mvc~{h%+= zCak1`n=b3{Od7j?Zabo5vqeF%8v=W6IHrd~MIb6FB+DcTEFzCGc>n$0?s39a2l0)V z#3GXMEdRfrJfeecEx;L0w_uz7CCp{qMzsNSM;&DeH*# zCYEgc6*f7CaR{`y17HmCEq=YKfMCuWlcB-~m@zI8$quxFGl%MBzWmbO1d?8|NMkZk z5+Km*`tcwQstN=-B*;A)2G9gBIdvLr)6ILZXKl*$XyUZh( zso)tq+j+w7tAbB&$DOT-%ma1PWDkz$LLie0-KRa2iVb>w;5>pT6=s6);c6*#IE(@d zYJe<7g=}$mNN?YN54U-zRM#2um*|s_kh~TOj+Fd0YkvEg7I-b`P2d`RvLA=wx{`mgk_3f3O0h$HYamT|mO(Fyuq)t_1jU6b$3Onh2@Nk7 z#uOs-dFWZxD5N^p+sg*6=zpj|-o*Ud%0)<)Qh~C()I3$yqnjEzuSXE6?BLEo1GX1v zS6Fn_FKL#fC_?iZ|C9^ruNdTxKp{}2xK#nK1-hb3o>iyOk(Q(YFF6wKgrNIFvdAg8 zjQAUUspnb>0qz>6h!}==*(@4blR&RF(*pRuL@7#hgUmdLTIJRO$nTW{WKo&xM|ZRRp5w2c34Lo*NSyv9dZ0%S`h^1PKvv^ zh4tdoT81uCQmAB=0J6prVHH=%5ZH-|k0*aN+}j4OWhJ|-cDM#)j677@QSn4WM_w^$7i~rosQXBPf-TM9s2n71!g4z(uLD}c(Y>ncA@s@2 z%sk)u{o?O!-@64A5#qaQ&{=J^ofc@n+$fJZ7(&%BG<@+zh^`OKwn3f(eYL_eftWML zxEdlxQG!$z4n)ZOq$wOJSDHYl5=F?vxec?bSm+6}$pxuZI8?q{2<|;C?}M((vHH+) zlmYU$ZbpKX-bAM<8C5YZ{b*hW#7F94R8MuqCK2QO#q*@NWzqqp!34XxRnW^d_&RI* z143Wu)6T4LOBow_Ud6H+@N>=73KDA3u0Ex0wQ8U1&N~MC}Kd#86=wsMUbEpB&mof3L*$n1VO+egGiA<3@ABC z&L~lm2!aR#JwN4~`@8qO@p_Ez(PQ-a>qsc7_P6(5bIm!|T6DCu^B=9b#lUl7CPa<( zDEDn{SZHQjyFtKh8R;r4RPYX#zNopZK0?yjelS-}pRhFHfIiay^`#azl^P$g&#z8x zOni6U)x~AY<9Ai!JR#2wQh~mS4Xn<&({@y>1{uG=6v{cA#XMGv@!Vgt6gsx;4RdJ4 z$skP}DCXf&bto;HPb0=!XWpzOT`&U_-3?|Rle}peIPbJ0S1RQb8$}kfg#&{v-06{{ zomsgpuFP!f(K^YG>o#Jvww55xy#VSjI3n4J(Y97js}55hV~(y?(!}gqaRT>eVHk&28>4G&HzsJ$pc$TdLz%hrzU?{YKia_N6dd4$*Z`kn ztqip&V>l_Z&@2Yy-n6!-TbNQmk!14;xP-{&H&c^y)l|M-)EZAh1X%IAx3Sti(;7c+O;+xNv0LSx7KRJ8eF>;k)m3}Zi2^vq`9_X--=}P zpMN@=(Z@iTS;s(kX1v8c$|&9ECv()pW&fun9e&~y=q8Sm)KrJD37}Zy&D?T-kyWdE zu7HyqtYi^8dfM_B^0=aD#?FYLQOb6Id#_&b8;PDyZpj@uP8PjCFiLj zP5JuB7Z2k%?KpB=@Axnk*q({s#043C16^=%ElNM(n0Rm&r%SWw*imn$#{Ka7)L2`o zWZRGK%$LtnqZMe&{sCHvr8(R75?R2T{BS=dCUAL2&tb)3=Vd`0nS|Nt5<(fvPTft1a zG+^OCOs;KGaqrp30mV%n=<3cv9-F819GZw`Hi!Z1WE?#v6-h?lxnyn3Cv*As7KYn*v_arOMk1E7#s z3I{l>kDkZeNdI=YG|zu{EbhMW8zM-!l=~j-Pi6_d-VZkI5;?-ny&(WZ)Ja(>DJj8v zOL~5b`bZ1h1z_mY;lcY6{KI2+?CE*ZrxKHEbJdYL7$S{*3Sb>MKrEUC)R*pI{ z9G;w-2^b7hfgO;X$;_Q+H*7MB+IZJnB-;rJYz1yXn?u>8!1OLNvMIO4Egc zgZsr#f*z>%%WIgv@ENzH>@v$^?B4GONEl2WgxNgtHkb;)sicMzVhY^qcks?%@Lm<+ z`2PKSB;RXdcOoe}%nnf`iYxM2tY`f^)M03JLACNckQvXgg2)ZDZa$%I%bLKmeaDV_ z@(mb7U)l>G^^$w^j0KA*2$seh(G4c|&+Rfn9Q2vhN`4Gfbip>wwZIiqB4TR2 zuM2jQjZ9^nJ2u6Tz0Lf3L#> z(?wx~i(vvnmDu*@$2#aQHC<3Xt{q*44`<$U(DQh#&wBzGYy1VqY|E7MKRTR3cfaK) zViik}=oFp)ukR4~H*802z4(nU`w64K+5G`G4|T%fJ@{<&ATs7VN}+zSmM88#_?sF0 zlSEw~Lhj-hsI0`6Vf1~1y7Z=Si=XC9u~ zBYE%lIrb&%lib0#!y0|-SUCReIdbd3d&=TP4v%Y2efIzQQvP_X9$QV#ojo{W*K5Q= zII|9~($^kfTcla|8sPVYDAXV#P|?ldj~aCORs8Bt=iiCHSSK65l#SiX`CM1HWbnZc z7PX&hU$?$9fAiuHKeAt%|6QK8<)Ri6FQU$5DJkq1``#($V)DZ16fmCB#e zE*0yX-G3>CE;UCNc8PO?REc%l{+dDle3`eHv`)6PhNNg;#&*#79Z@m$e@r^D(tm4s zWG{bl_~QCwpG0qS*KFMI=bwLHr^l%bv=-and-MMA{hzIk^PpblP1B`63tJCQ>MQH( zK^1xySr;;b2Na|5_mTGW-v8P7INg2Y?+Z@8ttw@%9Ump!X3iSzS4H^t_g|`)^2^?8 z|F!jd?Nv9f>0=7pNVlAG_|=03*R$?d+9|@)n{#I=(+%7I*Bd3Qx$&G|!Y&E>TY5Tc zN5A}7kY@fl^njRZO1Q#^hcfN>W8d#juXx;F3!i_i&FVN3MGZ7|+)g_EEcYLG)O~`n z`}6!NJXZ~8H4GOV^pN^-okSU0FImPFdRkX|!sGviIm$n337~9|+)|#LAyvCG>2NY< z{ml3GXP#2A{~*F6m+qhLA+%L{^1a0oWFP;9&>ar zIZ;hpOKmI&GfQ^Euq9c2Y_v$Ge>d7kx^M4a&K-Ikp0(e!H`Xj##scOhzZmdoe-3r~ zQhpvzXAPqibJy$)V;0}zQnsvmT~zO!h|O>y_6Ob4Qy(v4hKK1*%BAfuTptkoQ+JT+ z`{%m+(>UlLpVBkOy>~mL_>D^wZu|WCL${TNM~K_wFJE?wE#**WkSo6uZv1;Zme?aJ2@GamE4#GM-_!B36mi3|RJK&*_Fvr1fAL5EzN!B` z5hQDfiX#fx#WkCD)tH_-dj8E#gwCPplncn>rkMlG{@$Mda6)XqIr6}oFYHA93iaxM zk}Jza6X@wa$w`mcwjkeq%i`go60!9IYR$s9v=6)13%Z~N!K?x?RI}*C@D;C>QUF1B zn*ZUs*nTGteP)K-bOlHguvPBqVV?$vt5-v%++V-mW$O!I9D(7Rdll96%>zr@Npvn> ziNS4{mHR_=#X2|K;A_)1FEL$;jy4~@_4Guuq4Md$9(J{udz7{JS0E$&IWh6+OoWkt zP8}OJilP~Z0H*Q?8yxP@Ed!`SlMstt1r-))MWf6cFbaTB;t;Y>7&83IcaNkPpbezW zDkR;2Je&&%{}tCk26Ifl-zs&l~9hp2|MgCu6WDVt;@8T z07>D#p>WY{+xF`G+uOXaXD$+B*TFXR>5d8+mRE)ftE%a;b9w3jXr^#M$d#n7WI!4Y zsPVov3FwTTON;yZP<^OXkgBqmMX=-umuV6&FmvOwPnSo~X%%Mj+jxIN@**r*R zb;U9<2f-WqLCH6eI|P#wx}KojooYE5-fF$yPCjzs9X^u4ssnGQTVGySvHi{^7|R!~ zc9PPbAO(21SL)c2BPy>jz1^p8Xtu^&Zx%e)b!nbpTddPLfy`PIYSeDH&m69mH|E_e z#$-!9lPPlJE6gV+t-A`{jqGX{FeEl=!J*4lJCoO5estvTIS@gB9UlyvhVuvM56CLD z$2B!I7=Zm;@AvQDpJJ6a_ZR?xmU)(;_{z8{j48(^2U=qy_q{d!80^^z@aw_1of&ly zz$;qDu+{wd$ZWmO*QB?COea+q#Dlq944RQ}_N{%V0P%%G@Ft#R!gA!X#ShO3?1VMJ zle2s%?z`A|4t7yY&V^qq!MLv~Trcq+2&l49VcsvuuZzEoqoefpva_|<&1GmRwfM~+ zuiA*_t;mV2O~ z4>hd+a6Xfhk!szwCmlNeIjb^nzGlPVRb2NjUBGbHgkGffDC@hmlc-^VFF_y!8pifR zVfN5viJcd?3Pf94>w=gchdP_(vE;A=_snbgl@sr6@^vyYG9s;z`?5x9jlc0V-5&O| zaHXIYc;jIaJlRkWHpkP;>wxwhtq1mJ{$5H&OoP;QkH9`{uQ~nW4d#<;+#(d}O*ao` zw#u9W_35K?`3`M>pX^y*{U7gT2&0<`eeCJaQKugOGSlh+cHQ-Hbas|cL+&PRb~OI3 zp(;)J?bC0P4(?Z4ujvf>?-8Wkh2Ku+XdlY+#6d=HHpF=fC`E2tzWJ?EkIy~xi#p?) zj;bmuHJBv7<(7invS4DkwUOgz&p)}!tS}^AdelceNGKvZKQfkn*(VpsDczKIc<>Fj z^zr+sxa?B4#0X!5~t5kmSgjC z&&IM|G-^~m4+>VRZQvbmy-E?s4A=;v*!@0=#A-%Rg?NH3jdHGQ|7q{@ zLExJ9XCX1lZE;`tWhy`NbMvL$wemP88lN_$ZP>IasB2SrquZhKk>+JD*F+t(zljum z|8kb6%kzH*|G8x(4qR)!b4a^hJnB3znA!Kaut_mza^!4T&8|pfA5P7ia?N1<_kYlp zJ*G{Ya5`-JPD(-ib<}*W#s1x1`25vO2i`^Oo5LgE}?Y`->nr(=Y$^`8B7Yzls|OW@Q=277dK%1qj8ko$IACMfo$=Ab-G z`X%({_fa>!_wkWF9pt#lt)P{lA?stzfp!wD`z|+Z>x&~Os^%+G_1lA?P2MKxdib2o zz1(48^uBjv+csw!He_027mL<*tVhRHL$5f>&cKY;JugqiprHvL7D`dUJc4O8QHEg_ zQw6`1GgUh2_M=CS%sJ0OqeMN6pJzC8-+FrXNS;)i><}?xPVFu`V(SDz?`{P}%2|kc z@z!YtEDD;e?$Um$`nI*uKoS#@s!)J8hG|OH7lV*XmL1#~xxj9NID!pjxgw`UBvW{^ z_%&Vea7&*MSL~7miD~mw^&L%1w8+Fm)Hp9_P7QVTASyCqJX1%9!Q7WH%&*n4{5Z%M zzSsS?DlIM;DyD5eJHUfBg42N~j*5*Uo^z{Sx;Ae5z~rnJQPXo9^*RBvwz{Kq$zrh? ztz%jYEXCe&EP{!2K1~-i6vOMvq6=Fun2uv0`GI>Ks%$-@)aKzM(JAqo2^!Lk_$;b> z1TD(BcBN3NhHra#eD;IemD*~^gf#nP*=@edC6S48A3qUvS36M$n~?O~t`)b1j^&CO zK8`F+_n#dOGf2c4H*elN8gb-W&hm>U;)T7 zTfM;(w>XQ4^ZV4&VH zx8FNvX}_uM4c;0%H8rJqDH!BI8Tee>Dt#SiKYw1V?sUC*MwQKNmr=}}8tIh`r!W5W2G*b;berf1Ak6r=OA}9x2_Q+-fOxt!S{O9qrSy{FpI+z?C{5 zUIMb*bs6sTvXL>ByP|=|`K_(ZU$ndopxyK$#dvd}zsDH|<5mc0v&*%Y=RMo7i~F}d zElr8-wS1cV+X~Dn<%X@F`89b)XZAXr-eM}RO_^GAI05#~^Qc#aYB{=D;`zW>-nZ#h zd)BmqdPEA4Z}a&bRH^AT%kT1zE9?y9l9=(W4q&UpUS+8m=AsP|`O{UEf7fAa0^XbM-| z?20s>Fr53(#_IYvN!Z;*7NpQD1gWFLL{^Z4T8?-dXB@;95PI1h>N_jnN?#eq`MihM} zfH98cusAiTZSlrs_hW!Io~{^M9~@i; zx?n|38}^Nc(?_^O9j|Fq0`tal%Na#B>-S)WdUa6Voff5KG?8E;_fK3dHzOxXHaV%ge-eWnd-_PUH$@K>ANr9B>-iMD{F2WP8-}Nf}g_KQ2%M^n+lAI*q&kzl$R}B@vl9S<1CjD+Pjc90X{Q0LN zgpI+ak(d`VU^X}J7W;wkka7eTS*j!5Wuzc5)@dq+919 z5pkmmb|&wnlYHDacf}ZzXyJ=}?Xzs1J+OQHd|ue0oe$zRBMh(iC<`e2?6Gk70H^|8}eJ9{~#?(t#E# zsRe}#-&N`i9%zU9AMkQ3%t<7^9=h6`Bo|DHAn{e+>?SS8>nL4rrR$M6VPP05qs*@v zUG-YiW7cA|=9Qh?q`6{`ld(cpB)Mtp*?If*f4>~ZT6a0bjlyIfLcA+F^7KlP)aQb{ zzI7qnD5V1PX(4R*diN=c&kwDss^fUSPe+_nVpm<<_l{U7^`(W1I^(^jt50M`sf5N( z`5>`8YyMdt1ACju`RK57>x<-qh;NuKePg?Uy~(ZVN4ghzw$OdfQl~lY8{M<2o8fY+ zje-5laf>Y$w|1;e4_{7Ww_-OhP3pcnlU8Hvh}TEny&QU&(dPcU#bDz;bFx(|%}xCi z#p~a2l~6zDTGDB~U!!|qB8cV{O9k=FH~xL*p&ycm4oHpdbdskz#Q9%$-F_KzqswZ1 zJQ^C&lKx8l>OT*HruVLX1@->@`zbp%65l$$33VMRSB!OIaU6V*mY^R1(C4fa@l6`L zC+{fxV{gZP8GtjQ{dql$IN2W8B+|m&(5&Gyo045zM*QGDFLo17OHMmZM@~0NOicF4 z_>=a0R(5u;UQOT*{lVm2wYchk`8f(D_s`Fb;>F>6ONr+afKo6$oPVMI&ElrK#l5q` zDKvhjFT!UDSxO&u9^4ecazm#Q4+KBVX<j?2DVwVrqnS0Zgg(bv`1yI=nv@m8H5-_86+5_>gwt`{(O2NYq`)YeS^$}TrQekCd*iwEtl{R zzU(1yKR)u$`@H9TS(hg5{nE?q8+%MzRmutUH7zCDVH0SX{)$*Fwop= zmo8@B$j8&Y?obd(S!(`=z`iDhFcDQL)aUi*xVztDBuxe_R%~IodXGWn=e2mb(3$S?WGY+9%WO z`#9(reSv=z4qPxx%^=Lod5w`HcxS+tW1qF!#vkzm-kevdD~O`1l#^ro{`)VwFV1zL zp1~JMD+cm9eaG>)p5|i z!v{6;*fKAzWLeVRN`g*1IU<++L@1*LZ+o?5wZQS{vf9SaLTcOBOMHz;PyvXuH?KQj(}>P z&Y+N}!(eNqT?5E6F&;hg^qruR8g_Fy6IMH%&V(W8gnFGo2Sc$LL;S4}(e;QBmO&bi zKk4itwl0mI12&-YJrZU9%L87{RS)+_&IW25%AP%2+bpfIa6kIwwisakfOdFD`_vsXI>im$j6@j8TOY zK9fzK#d9Ra+WYR1$3ysJ$qkRt{2b2GJyGH;9Id2I^ECk{g$lDRO>m`TzhgXmQK*Wn!0aM9O7tls+V7xz z&*PEA#66ub7zLZ_DXaEJF!hC9t#U%RhqK9ewd)0bd|Gy%mntSt$`9u$L=0{S6Y>Qy z@##sk&O8viV(D;d;gv%~zo-yaHG22Yo{S29jRUQ#z1wMrI5ifCI>Bd7{bYEh(VIy_ z6R1TR__#tqGw!9$yxyf8*Sz5@$$U5lBQTIgL*kcqfQi9v-Nr5AOr!mI?o5zEc zHZx@o3AN>auh+}EZ|EjYQ3SIb%0S*T7h81XW?0cFQe1z3FRqO2%s%!m?14qwfCi<~ z?*23P1tz}mW7*0-k6v}7pO*(*MGuo0KTX3y$AiI0|po#AC(Reni&WXie zLthd1X#B)+HA_=gNw6qi)(!20+jNZE;T5^@*N$gv`K=&pRCZ(i2F*zicN~2u;5hi) z%ksj(pORJ6+cK=$8Uxf+l!N&o8W?yD z^+(rZ=HTJJ46^<2neStbIron{(_Z{1kSu*tpo@nb=)`%;>Fgu4eO)=XfW0qdMgQ6I z!X2_bc1HhWQ5Xut_)UGSgdE27uP*i?nLotFmT)U1KB5&a9*}Jnl3^|_WmKKRwQw&e zcohEb;|=jaZYv4%N!Al?W6RoVrmQ%kyNtjmG9C)+9Zx(fp^|)us>6b6T9v zDkT4G!7FX$u0>-Xi=8`I<({lPrj%z~zJ0PpfB5m4Jx8s>swc=O??Jm6#hi)cMrEvu zt8aAPX2~CZ+eT2jH~(hEW_{>Hls>p5&zlp@4?n-|^%7QtNKT0kGvRiJS!$lW`X?%` zCJtDzNh?&HC^dpoZ@*3=tyr^3qn;a;zw5N&9=v+2{1A`LWGC!xahZq!{br+Ru9gxn zXjb#4g_|c}Gb%9Kle0Rp(NzNm@nP)-!W}{m{ft@VYY&TB0{X^KDyd;mCFEazM^LhUDHOH8#-t^{ReW>E-Dn@E;&(*S6xLs6z6IdsxSm*9 zx^n8Q_&O0=@V0h~F+gb(LVu7~7zs!WI9y_X-STJ0{{X%d`|!V8q{u;f4J(}X{zZ;| z-(Hb2rSI{VC}8 zYHwJquK@5C|8q~c@mbx;N=U`Ug3`H;7dib15p_56Ml;*m8EU_F6)qysI(5HU=I}xG z#XiQWUN^+cls)Ao?2itllu{oBakQfa|NFhCIFIWhm!zaCJa?s~ zrB&~M8s|Yy)TxQD2R_cfhx4|aEacGVNP#qBhWCjXL0243R}iKlUFuWA8>Kqg?u%~8 z7g+U7Spb_EfwYuG8u*EB5%nk0!RYw6!e9NN_}A<2ADLlwJCZ@jaYTPMOJ`X{Ma}1v z?^FxEeR@$&rz}yMVjwzm2KH=3ZCC@>3q=I&&-2n@b*8OJ6>{9timj3?KxM2-Ge z?hT?h?(nCYU`M-+69T93fAuI%&>SGPXC{89Jv%#AM!*F5Z4sNtTt*kadnwx!frwHC#&IL{ys3{ zwup{rSj}V1>&34RJ&|%>NUsu0gw{=fwpOW$k^@XkOq9kzOf72FKz$lcCHm*pJt+ZF z0hozps-yoQ6LAc8s^0JfZV&{{eq{TZ2h(d0kezt4Tk9dTUwD64*BhSd#*eK8AWu;H z3ANK8&xCptyo9=yn%XC@8p-}>lAy7v8Gbz`RVTeZLd9TONkcQvPVD>3tzG=1xYP}C zQN!9E7lWw5dDOI;5*tgv!KViHWUJI8@%l|*{rHI%jtSqpfRDkvko~hdZ^*Nve~FMs zl|mjuHoyyMby#?$OcKZ{?8QpKI1-73_<+TJZPl&z{Ddb_#k=V)lVmx0FG5RyGG51I z`~mOlnr>)!6_KS>U4HBzf*_LHNra)_ms{6bYwqPE zy2&*Lfr^x7B?1GJ#=iQdVUfC&Gt zAsf-s+~)m8VF4)$iY7S8pg0p>#KgqJ4G{(b*3V})OHJbG)hE0>;QcDjwdQdt71F1W z$HJvU{DpvKCPx$~go0!8-hlR0K0$ZTE?u;^$$Z8ie>oIF>iZWN(S?^Dz<)eRU!f5R5^5H7Io zSNre*k-AyBC&VH}H61>Nx!uL2SOv0waIhnS(ELO>Z`H?d`IEjoH!urT=1W=uw|bb zx#jRYMH9L@hY`P|G!Ses{B2kCk!OHMtpIv-5;j3_|X7VUoI_`m$|zEt=5M4c7jj;{t) zhX`^ZD`O{Q9@v7pi1*D?FlQ!es(&|JT zB6u)bRx9<(5PI(`Rvq7GEft3j9pZA+j#GJiOx>aSnIs;H;J5S&s_tc;y#2Yus)Q7R z=lb%|^s=!3`4qMqw<6Fas;sM#;6oRwNa>LCpWk^BrR&10Xz9&B5+`3b)*E;hogHB&lY>;~KlC%=w;N;;MOpZhb+j;!VCCrQtN z+*l*&`m`J*c#Tq&WLn;r8sBk_;qbYaZ*2;Da!L{$J^P@6y`w9QYVC_lu%pC0& z!%HtHl-8gdsoMhEw*C6-wfD#Kp)VORJ4cWxW7CP8b@o0t_1P~+*`ZaaFFWF*<5mRa zn_(xGn4IqcB;|O$1}-V>c(v0P8VqdJIE0?EtDeYdGPpI9L7k@=<@8CgX{%128_~6E zpRYb4_99cAreytp1@+N*3N9dmk9qn`1A2Ba(uX|y?^kR?UwdctPtCvBeE6G~+2Y3< zZI2cIUWznP#+OEGiV{~dngw6e^|;?8k!A9x>1*E4+|cDT-MjyuIU2j0lV(<*NK;-< z3s^XeJXk!L=N+3una?Oxee3`v68TDCE8Gl~%?|slI_=P8Haw}R{g8O;ISa%0a{Bxe z6?#8R9KQ}L{kWn=Llbt5c-b$9>L{zO3>QCOOwFH`6H8vGiSntOO&d-aCDs#ble*KkU;@qaaVT<0+Pp59GqArRjFZ41S$uUV47jz&K@*;4;y$FT! z!e)mgsoGCKJ{&~RM0x}fliP-U~yvOkp8pS;P9FIn`bh|d}C z6-S@Gp3JHp=yl;bv zD|NPRp^-1W?jcl_e_#EU!uoxj>3c{Po2oZ)IgQ>9+S^?B@Z?4DReu;Q0=uu|h?EiP){3`B#mC@MqFCY2$p2XtX zCEnQg?b3hzojWt71e1Jx)Bgah|AAfqZ~yoIb8s8rGkOggcj0Qww#-TchoN7ngls=e zhRP?6CN9*uzLevC2$)boi~hqKStoc8w^azIxB@*^)gCK2b|P=$28P6SjkRW!>o9v5 zpPIfL-Qx*5R)TmA_DW{ryCocLz>~v2HXu#`kP_~=RtIjQLs5Anm?(1yRZ}L=`B88L zman*jXBVJ#jNo*Y__u5>zpIDy$KL^$l^g6VaPy3U? zjr3EzZ3%>8_9>>cYhhYgPaJ^)1=T25ToheCakoU=aOHu&>B;uv?J6#sUg~-FRs*X0 zdQk6Eb}QxGc>?8A#u3zWxF&+v4ya!sN~?(Kf9=av56IIW7>(*BFyEmWf6H(@nN-SQ>rlVP3=xZQ#>qL885 zu2GCbmAIUdvd;c2Tr9Bpd_vpm$z%%%UBLUcJEUG1nry`l8I+=CmXuGaz zqRVspmc`O-htNux!FH+m+%UjoBT*7OBWFPO(06f;f10RNsDq)@Qn*Tz*X`Z_G4eAlYqT_r_ z2r+Ut8y!)Qs8u})of^??#-B;Bf#+bMg5N4QB1>w7_WK%>e)NXZCU_pfLn#VeS?zZ6 zu-A-6wwjfXLP6zOmhbx~-P(S0erd-rvH4uaQ$)%F4)mM3d{}6N>rpFRog@T_c${kE|uD zP*Bh)i@!jxdgb*mlYkA@xnJ2#PXqvm8rr;F1y7HFhwfolTk~jE$$_w|v~WBX9B*qr z_y!O-1;N1n@hC%Zh?O3DRe)|dHKxhXIhogHMW8-gDaH{U()uqODw%Zo(UURnJhvZB?Dn^zn?`OoS21r*%SCNWvTKjz(&jJyZ* zjNGyQwV$Eu#H^eGehqD1Tm7|Q;FUPB?0nk?T5<-qt%Ovln~=%@jYew_KGdl4R0IX?z#5{u#uy)Z8J?WHIM)opmZL zhOD$FbOT^N(>Pxj8LoS9U=LT}uU)nBP7O};R>9<-Yc^d!42C{m`#cw%B!EB_k&EXj zlK7+E#U zjF$4$6x!|FJM83_EpZPx3>T}cU51mh+8K)Ld|!uD1|NMN#C{0uzG%PeK^795AGj^_j{AmbL$_IqSuEsjJhA{>un1o5y<>dNJ-$0|VYaC-;g(P2)W~cyZ3$jJ5`B%@ttPCzWz-Lk1wCT%0u+V4D*bSkLS$);>}fY{tj7d z8H~&QVii~pXbP2DShM~gRiKKhgR-M)ai9eqt0jLnuU=s1OP>9OY%{N4%c;8Fyw;^S zN(q_URA<=9VJZ8lq>Ili^qt2nW>yY9;)XNHYs+4z-p52kuQaXHt66-%D|pcv3GMXD zZV9*BczhWH+cUa8lvKHmlwx^beulMd;sJlfXD#snM4H2hTnO7!o}Dm_9XkWJ)usuH zw@8Iwpo>Y_KuWrQ6e(YV^(g(kXw2N<|0#Gf@>S&qA`99v-RNi3Dy0 z7cEF2tc$UA4E?+d41-!$=!#F0*RL(v_4?{=m-6YHVz8AaxG5@GLs-ILkVdc|I>8~b zslx7{cg1(4#X`r!Q~Bn?J^tk9nzJDN5(=*Fc~|ip=6ibjM2+~1sk^lNCOs>uCim&N zt^QdELaOXaqoac;PWg#(Ca*+&+IMMP$9oopK45;ODyG>oQ6F1A!eNGikTeUdk@sHx z(XAlvI(scJA2dp4FBt#YX$$KypZtCPzKmp+zkq&=IFdhyc3Usr`tL_gDGgVsyn`bC zFZ6rMhs4gkc#0=(biF~d>cmxT7H8X@b;i2y^_7VZ0sK4WUA$j;`LrxVn#I^e?Mvn! z$d7DtNvWiIM=ml3xh~Y4+$`BEZqL=>*XjDxz-Zr!Ip0E~dhnv#=;#QafmyfH?mjau07t>ms&iH74+AOX$H-?i69Xe55@mop*3uB2?D|{hU_@TSm`tz6#-v zO_hDFT1QnLUWdipFbbe&#=w!eahFJqM8U_gz}2R@PSBrSZ&^50zE$uM!F4is`f|=G zmO>HMMX(IP2h#9<>XZ$nXaWu*-s*IvEiP+8_kjbCB% z2IR(w1_Eo;?tDEM^#FNHtWJZNKJp_(Q_%6>#E;cshKR?kQ=3_MP>|Nk^Kbor=Dm}p z|8EbVw=MqN{pn2CTd@wDqWZQU7)--#@3sUAlNwhljrPS2deHBnH#4|!VD?x2 zGj+HgK~7bkx1n@^qs_VMZ~Jx?qYDn{YfEEY_iiUqz_YqR45+BtFdWUB)4R=d#Z?C* zXDLIj0|$Bto1i(je)dZlft+0MY!spLm2odrRv2|h>Ca^!p!+=XwFOB;L*%EYT2H?u z+GP}^zS?Ti54yQ9QU7^4eF#4Zr<)IBx!u{wTBYg_NR0_(;o4c@44uQOCA=x9@ZYK0 zHG|A1$>;&){NA|eh=Q^PnerD6ae+2?WELugYISeiN3IzSbUc6f56DBZ;7exe$sJA^ zhUKdIF)1TO;0qEKvqNzLiK!^{WD*@4#09iz-s!MR@BVL&TOTeV@R6`Vdoi@=#?7uA zp9%l~MSw}d@ulPfiT#u-Z0Q8FweQP#p9c0>?>eX@bbXA|Cv&E!U+{t8Nix#s3GnyV zkGF$o#hX@VEE?Tw6mm|htE?7MHobn|?O4VPLS{;if_9~Vw$NH8WKT^CK@@(ontcV@ z@)@Pmho3fj^sB4ag)kj+$sX?2Q`DkUmX)>Lu3I+#t#I-6$I*twr}lFrbazdvpN+au z9-D0y*zNp8^GO}Nobk4!u9?tQp9@WYPu4ZScKh_C0dn&(@3;2)TBFSfgYj?;~Ucq}gL zMOaKgGEdKG77;Gaf=WTV_X(T5DqM5xGC3wY7%qgH6|X~`&Dn11l!7%^{g25~4J++m5cc=0Dft+?c!40Dv1 zXf2JXS!rGbXLLx@&!@*;<_|~CGd~jF+VKy`4e|zL)>DplA=0O!DVK0MzDplBC`vmw zjM0W&v`pIVZ`0Skd}Dx;#gXqJ1C1MI#9SQ)EhZjYEs=FJG<$j^bQ7D#KQ*!TB}&EE zMDP7}<7&^n(w4kX`r8Cj0j+c|{GscbPyIXt;N^J@YYY1&+gg|`rgrL5zu)7(!nakJ z*&*P>k$;vfV0*UaI%2lt_p*j2zoNv-&?TO(Aj9^=Keu|-w_=jJ?pxyZ@}u07$HZN1 zZgI&36@CaOO@7sMf7ehQ^Dv@k%+vPv(r-ZX=qE_Jr?1_LzBc838mZZ@J^w@PCm1^W zM>j2rSoxoP(_XN!+ zcZT0rA$RPT)F;b@)wERgg<139x`$u?dM5s`LVtd0+P?U`x3Eslga@}WKjrV^C78M9 z3!u=<4!R3y6bxoDcF|1;^!P1tuK)gWTc*YQa_(Zt1hxF+%_5FW-)*E1I@(`p^?PEI{`j~VT9>+fv zX0s?q+gB9lAld$ud)>m}^pzX9(WaiaWnEpyeDOK%>DdA>F-WQp<^7zx;yl7K8$v7^Zkl<#O36 zBDzf72iN1E-FqR;NIqxi)|x#v--vdYW8V3np|zyCD8TZJC`th-hrG7BW79Icfy=Es zme|!T;yGF2NFWkl-$d<8b^1J@kzhZlRV>lU!q5)p z&U!^Kkcug->O6JN*iS#Wb7!RkhwszkCN&${GIa$C(F(4DsFVTqcp^|C_{is(yU#gJ z#}U#+2ZJr=OdDJtH4NJNpUoz`VE8$FJwJ9Wac&_fEDL)@^?JT(2%a7eTxq%Sct>Iu z9{i&y|DsyFLh}?&T%g~uNLiXFk8~F%x{qYmoXdAi+nEJs6App5#R|j#CG?|dAb=O> zw=}Xnw!mx>I_KuLq9g4BfS?Wf@8J)i$wx>xnlqzj*_^hr9{T8zWqd23^8~mqeh;t! zs?1H&A0ST7!@$*8kKTG{*ejry5@rKLz{q`Wd~rGak!Fb)GEj?2em)bBQHZA4L2J^7 z>w-4yJto1yA!|)Yqp`|+6)&TD)!lc?K}ksi^&@9)Wb}g+jeCSwiZH009_vWlEvJaV zq=q4{cpQ9FG?Q1ui*Umosu58OV0H6#I7I$dRXcN@(|qzFO!Fp~Ue&#;wxpzF1{x_w zSEX_cI*3>_a+@Zbs0C;S9{Yk7ui|{#O&>Fjs;Z+8qH6^sDEf0>LXI}o|LBM(hS))E z-*}BKYAbB~7232x;(HOD`EMj4L($j+mSqDKZ}; z;)B11GH%ZPK$17eXy)ctmE&`yZvT>2N+aL$$9Ad_d;GzZU7L{ zhK1o|-)s#oJ7xDN`OJ*j6GUu711ney2M@&t!K;wj_#u!KNObT?IQ@F$Rk?syLV{sI zZfP1;a$@&h-ac%`rhJZXztUT`RICA~SrNAVxlvDqda1q*r2t*DGzoO7=ig#&IBC-O zOcgb9b%uqq8yCO|+SBSbaHc(Pa``aJYLOgbixX)=D<}mfAS}Ee4vewcM@M2UuQ6W0 zj!5kXQRomfXu%25e)l!EQ134LV;lRI!qVw0|AGMmzAVCLg9S>V1jnxWeQ4%8ThY9Oezp{b^ty zNrh5JkJh@hqap<^il~y&T^$ZCy2L<+=^JPhWW74vi*EjTRYC`bajFr9G;?|V=4u5iX@7~_mulKUfwp7Mp>q6GG6XeFV$9|rhuSne#a@3T^ zZp+`WOJmJnY%(evgPwx5k&2?yT&0KS{kC6#k9 zXC;EH=Lz2!)>KDCr-70;QIf6LirGgbHbpzB-=7Yx`}Aq3 zDK%9VKGfpI)v+0%Z=QiEtvWS8AJzy`QNWY%0-Gd{pD@u4B$B(#A{SnJ-mVWWXLJ{< z9{zy6FJ2|Sd9TPNt#8Co6umi|PmWb&>UkOM8`s8KLN|P!HB3U7J(&I^ICyY_0qt~I zTK_yIAiCU|JJ_Ooh;DhTa>NyRq?@L`Tkg__-43H?(@ZQeUCk_L9^+m)n)3zrh{@e^ zdxN-zS58WUy~^CU|HAe&w-%p!?a76Y;Ku^Ilh>-OX1ZJi$eGCdLPD%A(Ppq&RiWM0 z3%NaRGZ)%EvNCs^p6OEay{ucbVQFNI>GsMRB-L(~@%I`@{O_~ebC zw|xc%c|PfDxbb$lF3fqvY@&GOsAl-#w_T5=Y>boUX)8oaZBB|?=gL|ZDRz0bI?ief1_lnb zQ-wAI*IzCqxlPtyZq4*fZQd@%tBiH=;2oD-Xc=^TEA@@LNo}TU8@ch1B@>=Y$KYur z*IQ;yy*rFI2bwX~bZ%i68DIG=2DQ~)BOa|^ci4M5VmNZEXG2xoa{p<425U=->vW5$ zV9KzpP>O=ZexfXT{#G6Rda(TcZx0TUFoNTA2;FGESQ~o^dxb^F+ooF2_CPc>{E5JJ z&8Yi4WmvbYo63d+*&Pkyu-TJTs~~{FAfLBWWL0YQu2L%KbJgcM<6swMzQOuFtjojp z>CUc>y&@iPS$wN{${S%$$RPuj*RixM!K|hf!DtJ{mXvWt&2xwerD zDT`M31-yfCAybeq*gFKTv^RIgz!b$NR2MS0&1aYBMD^C&e98UKp*5B9Bf@{$lU!hb z^dReA?R^_4r($%}u4zaJty!JiU0{FmsHXTl8WpN3r-a+-?Fnm40x+DPj9BDWiM%;o z4pwF~kq?#<7?_XdqK-GJW-@q9Q|*Y8;DEcZ{w2q%Y=QPq`LIJ8w`V+yWn|3dqadZU){ zEn$amJE+ayYRjePP&?RL=?7ZltN5`DF@7s>jT(s~rWWXO80V_h^_^VQ&e|}NY+~=g z-UB?S&D_NFL8*U`y1xv;3)@RI?cxV`K-={;c3L>79Z^Z1bUN6ojUDSAEN`l4wwYh* z?J)ltzT_OIM>$zRI_#&_O;>RB4jzW8XbcE3vf)%=(5s{JRSFg|sp_%5lWrzjfJT~a zu(dgyX4B`019LFTt!KfPJh{IaL1%XP&+VHQ_68kUT8!K?PyCMq*gRn1qU?Riv2HO` z+fT&`z=}mwrcmJXLn~qCaU~Cz&<2yURbQ!-*&5#)^6BJJVKV8S@$=-S^8{ ze1B^Eqea@=*^3KShpuzdwZwI*DF}VbJ?`38yF-FN-JVVQSA8T&I?kp2qEuKF?kCX8 zF_6%ou?B|vQ4dc3VEHYma|#O%K9``uF?oJl<)aB@phVYVz)hFAzLK;kX<^{akDzP27S86B#`ckQW!C7mSJ(X)B@;yLgnOWJ&;`K89>$Q_b0vUkf=OD zjMbM3efG=^&c?V&hhzEmtd6cI%J{!p`wFP6_HSF%V~e1a3W|t;lt_b=h?JCqbV?&7 z-6DcWqk!N?=SO!-Nh94|(%m8b)<(}6=lt(Icf5DUb2uEv-rMinzglaqIp?C9u!r3M zVRaV5cCK)K>$*^nvQ^k)y>yk51;1Ax7?_#6e3mYJ1js6wUc=R<3~HXR`t;th7XrffrCmY{Cq!IGW1s?~>~qDEx7c zEU9(yqAif+`X@I1*Q=>ITvbJWID&-N-URU_j@>ncE=7?W^m)81PrK)&T`ajz?z&7E z)Y>KeRXgY#7WXB3I#RMAtK)8IlY3F$n0rC}R-lHF^o`K_kC$S1Pf66#;6{E|vFy_V z#@ZB`4wg8BZ)E2i9T0ngONfFQy(Waljig+gCe4ChlDaLHGNU-hFWOn%n+K93EuiXj zuXA=Fq?FAogwu21$tBUjD(>cV%u}SvU4#bfP;O7!wa9WHTj)PT2+DLHF+C%OG%Bv% zV~RIMfl*f3LK-PK4aatmrptllQ}W}kq2 z@9|3jVoOR@&ND!!`cor@Nmcu|nbW2UVPjNCh6kI~h_9!{*XBhXI_D&Jn!D-ky*iSJACR8Pm0+nh&I-}hj z?Ah8@2_)Kh&NI1C6&xS!ATn#ki z?6Wwz>zyo*UVa-K(S>;!oWeza7LjH!YU|(m)%u!YkcjjPbZ~?{zLFNfa2^R zkXm9;%e|Q~uNI#(y#Ef*dK6>6G}vcDsU}oD51z0;goTBD$jf+Eg$x;r-e-xu-5qO8 zK@{-KI7z912~epeJ82t(H!C^SACA@NXD0+q#J%x&lvJ&JQpiZx=pQMuCo|?NkkYc4*Pl!_>CMw*PjdbGa*&1gC|WVEFIi~-PU!Z1Xi^U zd1z3E=gNsEutJ?9E1+P*elaX3yz84qf+J|RsB+2_9m&gVG@vXB5LIS<7?xW*TC-Hu zY~LPcKyak@^>S5{y2pmGRP?nud>0OF6j7;hG>#$)-bwHZl2IvdPI(A7_m6%~@!^k| z*<(2JwS1h_wp#I6Rr%&nrp{j8Nu71H{llr5f;-bG!38L9hRPtMcQ)6?${MTyIk z_`bR$%LPo6Pe)i6Ice8?2+IFNNyP&JCG#E#w7R*m$ZwU5QW4%EOga47LMjrjy=0BM z7KvGxL!Sp6snh)cnhYpASqEs?UsX2s7mPWi%OR63deV` zI$6qn*Owx9;m++fV_;x_czpy^qoeu+Kd?fnRUn21BE+Dw(;)D{uu@kL@P-=Omto%e z%5lXX0gJ4gflmvZ&coF-)(>};5}7y(M)Am-4wrLwV~$MsjFTTLeLI8ksPUJu<|Nzs z#;^$2pi=Skmb#fWj!*hD(pP0UKQUN#cMJ2xHOp!C6F(h_<1N`b{Mr|=pitZ&aPQjQ zsny;~NrOz!M5Llk#N5#}Oii~z=Z|*L3%}6u zRnj1|p(y_vYcir@@7srER_cmt6k7(UzFT?bMQ{@kI}aBh7KD>WBXm$ebw^|S)*EVO ztx8v?wQ|V2$yKz}Q;V1}FfxDV&ZBo5dL3WeOgoV_3{AM@IufN?u%5=a?fUmYB6!0w z#9*{laK`tDZe%%=g>qzBVuPgn<+oWnl0Z(B1M@>mngG1~(k52!{QD)A#;~=QR&;5h*m{k(CbYQXW z6mWA%662A~KQ^Yn(_sHXvk|HHZ(i(isL2gH%OBvE{%&7W;g})tAJo@l3?0$x{|TxeW4J8VZWK01t9(9lSVtS_kAZ1lG-eIeO;K@i%b=-*QPDln*Cc2H#Ox#U zyblW5@N+OO-VZ|Z6Bs{!3kpHtBvqdST=xuT#Cqoj#B)h8=S|-p+O&GyWP2S6j|4bK zR7mLc@gqaVL7Qh@jVD_mB0=_#kB889+217Ff_n=@u>gwvzNfSla_B+|bxxgtI8V-5}Q z``#>6ZX4f{Hw` zfJ3|SnqqQF0<#P>aex?}fKqY-G@52U2}uJ+MPh|@R-zDq?QSm{58TUBhR}&B9Xpa0 z5c6TkT@BU5J*CgR=*`BlS;(w4a0cy6D5#wvx(apzmqVbvG|KE`2ipVz$fXS2KcE$K z#5S`}6FIa3SYnWxwF*-zy#l1@vJnZ|9zl65`}VUoVX zae52no@pmzDd|l_&aVSz4-}(Zv?J)`s3pKqbQs(k%Fk{(^yxn{EQI>JGc-8ki(m%o z7-NCMp5?SjKU|+EbF+l!!E|ueodm9h&`Epkz3T)G&KMY^w8Bq#Ukr{5b^Cd8E4#nQ z);ZdH)L=9XY?Tl{zvwoE&X)ZXxJqp|kY0*mG0;z2*kl`osXh7o5ThqQaa{a^eft^) z26$N@Z2G2v62VU3qapwTbxbMHdRb^Fr|;dMWbNPb5LV!S8gE+$a@`U{VaOMLL7!of zpFla5H7^5d+f)mnXc_js&M2^q%wC?_)je>HJk8;7lv%I|z)1g@sg=8Xr54XClY#;Dd2M;ax>x|kv zxicu91e*wz-f(fD-bWZWjt+sWK&8eY1~4!<{xVw-S$C=>%cL$;0OZUJ;Q!P+MaHk| zF<>JZ>G$E5ZCxOQ6c{Vm$LC1DXrGzyXSw?&8hY;aNVwd~j1q@(2s7gOAM*zy z24GHBDM6&1iv=W%i$Ixvk$@eyedGBu7_BZFfF_D3VFfTa2Iul1+pk&CJlHExV2G`N z4HU4yu0S33^>S#GDi7$f_h%l*H%-Gp1rse~r&0tTl%7}>7}Jj14LgmNXm3s0LM0gL zBxNXy72UZF94P@S+Qcs+ubc&s$3XA*7?`R^Bu}8t6@ab=ILH@2WGg)>D&)Zjs)8g_ zH5kVQR1JJq#E`!?n*U-6YyQb@K{moKAI|rAGG;e=2-w?j1)Q@$#rn=5=lJ>9{uW@)(Ha zFjkm;*8t7-Wj53HZslVtN!cQz?sBMlQAHr84yYlYy6;L82#uBT8bb%zx;%dm!M!aH zkkU(Jwar+bvB)5XQB7)r@@=z_aN0|7#WHLw`TBZCSOrA*2fvAz59&GCot zTh#!7FmV2edm0QXM>#z?;6t{9zv;*-!S=(`A0&GkA#ve%2GVyHX6QRXtmYh#Ntl4S z&he4|I6M&T0W&!A z0*V15y9fF5lA>#;@5@Q=s2;8nlpKNkj9$D3!y;Lbk|cmrFyWPe#1GA6fW40p*bPEp zol=fbaX}BF4A^l!UIzQDABxIr^OLQ>_R<9t1IY8cV^I6gbJ&YG5+sDJnhPu-v|S-H z1YpM+3cz43I?@*jhrtbH5ulC9>Kfqcx35>rIH`qxe9Y9WcyWe5ykAUVJ%JFH@y*-{l%Y9#1la!RRfODGTiGO*(%heCf zH4MIkiR|Z@aHv=#88n#1K{j~!Ewc|eR1mdnLe)mP&cs!^YXeML~s$BPB}03 zW@@aAmU~Qs4NI|UIwPSX_!P^iNjXFDQ}LiQ%$sIf3fjW~y4*sbV$I8FaM2kk1j`5! z8d_wnyDI+ZkrYx`1X%4m=q;kt1!8*ZzQQ3S_B2~Z@k~oGG)NvrM9-nR7eKY{G*usP z*0Bmy@5`*8?n)o!xQjT#MyT7z#^NnFV{MklHK&4 z@^6p1gukVQUTVR4_!w`Ck25eOErmsCiNd(|!WmQ1@;&o=4YxQ!)YsXUKUZ=XoXO%H zuKfzkql}-T|MaJOa`S-MnTi#&vpqmpNveVg1txa_55yENf)f;?cp^z-NX;~?-v^oJ z4V5{C#Yb4e#q|r)N2~R+#_Ik$QZ>!>HLG!hY}(aKVyDnHbD|yH!1dZb=8sp3>g%4h z1)>(u@aEL$V_EXaWMjINt`Os9O!%=Ffc!9q|9VsEzP+`sdd5kRNmw<%{7!y8q$6k2gpLNW8lcMiXm@<;&8D5v! zZ5|o4z%b=yLFNk6-xg^NzzIb`2XpsNf4m4u=t+#M5C6u2s>}HgQj|DNM{w#xV=8Nf zL&?&2|FDx2s9ZYm({n0baA5>fAB@*?K;yAdibb(NWqFfW7XIqdAZ)u(AFXQuXEkvE zg$RKd0+b@CoNYM5zkj2AR8-o3qT49L(iLKm(P4gYAp>VQpgvoJWekuz5K?B|;t^bn zVIsy+(!betIm6O9moe1OZ|u*j$tfsWz)Kl%GfM({=Ur8}_;#9JyzF1kmXeYhDyjnu zum#P2$n|JcOO*!JI}@b<`r89E0Ry`yrz2TebrD0KRgR+)5$-PEImITN<*x^K=8}Ot z8<=!*gj>FF?IWcQ@pMav+*2t-YKxZ;8dHhqrO~_J#nD|X#dh09hK-On^(Q$t5y~`& z^OW$j9$3g?)G_`}@Xb6dZFA?7Z>lYx(br*v^y<%>n?isCcodNy{=fIdgIWJqlKx)| z^&r*XHXbgGH3as_-#*WYs*+z~RE)P$;131g?{6M?uojTN_^&2<>=Amb4qO)H`dk`E z!F!E%ZiQ9s=BbaQj_^73+bcVZ_e zBa4F$qFP$=|9;P`m&m#kdreIZ`Q5vKQ)p>xEByP#KfKyVqT1TpUe~~Xl8}&4P*4EX zNRX=j40gpt`g{O{e8LKuf)3o4ARf>BRDKP4eay-XxWJ(Z%ub;=oda9K+8u0=mW$dIpnnL5*9 z5C0xC!;;+K=Lb8NH&-nVd>yN1Jd5>fyne{7I%B!cdfHz_UVzDropEU&W+}yK!5bWT zEp)B2p`3+y65t+4yuoi2G@Ob@-L`(7KK!j#|G}PYPm(m=WpI9P681kM@I+K0Lv64% zDjh_bSk41^w74DSLB&6edflOlaM7Xl&acn1nm6z?CK5E*A`Z{nW?hrNpaQYSPJbxf zuEOyiy*tDD@quEVeTq^cgSDHZl$7k8zi~7;UU*wagN|BM^~J~gDuWh>`*gdgac|+} zZjaB%yO&U7SVS>$RP|&Z?D076m?mlIX-}c0;;&YZd=&|2%baYD!VJ2<*v&fcTe@$0 zJv_l~JARWN9~t`BlJ13#Iq;4zRYWbji+XN5tUjQa_8d9PV;2A?sd$M}_phd8n3#?E zBSvimeMwf!;?~mc?w%X`)nc~!GAU1x5kBd!7cWM5X<5I23Ku(<({FrOw=#V-RPL8eK}l>%s!&saI%ia^F1vU zLXiJ<>FGX~lnkttJ6w-2+!csb^Xq+~Jw?uLI@@maGQ@6MAc7-bO0_zgRFJ!JA~u#R z$?-9lEvMo$bhUr9u~Tt@mC+%!u$Z!Ykk#tgeFsYO9aAdu(LRuWq(vl6Wn|aW7AQ8< zxmDt?2)v$ZyzG$rc=9!C+1^q*+tc0opyPGp^gRdjMyKmGzd&g+U6zoQ@Bv}CKSjO= zFQ`~HeQymV7{!`S78#Y5Z0xL4!eFTas6(F$FI6^u=?ad3&jOh&0gbB4sD2DArP0`{ zW+mDR8)t8FaFdXq18sB6mSJI!$HgWf{cs{0md5j!?{6ON@n!0kzcaBKmKRTfhLKzw zM5eUQGwimWU|w5!NAn#mM$CH@YeXb0ZEthun}SBo#_B$2M@@ATfLrKL=hUdsF7I9s zX;HUNY#Ur$9prZ#YlRM=AbGBM5|EZpH4K&p#1eFKToh)@C7X4XtjrY9gYMcl~ zy8L>SJsGq9@DT`6I`xDWU>yhDr+sveK;+_L3Qcs94~snBcmKHE5&ir!^cUYF;t7wW%Y`>Df-bQ( zA?Vj)BeloGr0>_2Q}o35O(NtS@Gb|{#t$sEpP}q8L2gwppsvA#jUy=~>!EU;8kX_9 z!_D-Q%Xqvxche$UpW?3h>9xIKHXc?RE~eMhd!FzqP1(^I1^C|cX}7Tp=k?yJv0E+8 z`0`xl8-I5a3yb(|pjMTeN&}tzgcWUN&sA*pJff0hXT0a>?fhFMHC2Hu>j{x9!Ug5-mBE zt|cCy!nydWDyypa>oQm30sda?!Lno&FQ2RKC^Cw52<4N;fVIQBSoHQ<1XD_>q`&jJTivzN`iCsi&D-!wHn+?JjLW@ptt znMg&K_o(ujy*nz-l|$eYds=0Bw0tNtgsph4`^E8Qi=nAdKxmL)KC9;N>HZKIEeihx z9#+@3S8J=$wZ1~LLLwN`PiHV+HF(OkqBS&suG})c+`>if`dpXng3b3Fd@XIGibhvV zVwto2CXn~1Pughd@^bD(BsRP;8OVEo&1S1l23Ab<41ehnx3HJe*Br&rQjjWrQ2(RE zhvLZ3ZgaM^zvE+DJ81^j5 zS|Ashy6BxiWlP$9auz-B3ct@L+y+TyvQVh&`({Q~uyqDTx51Bi8aS6U+Z&Hz) zke3Bhb#xrP`!sT}v@83^F5VxlvBIb5+019C$6g8vshrBOm_KPD&fC-R6e!2%M6oG& z?2f|s@xcmofcxBJ>E?`t9mNdWNBBji0w0|*7kXh3dOzL@-cFB3QHHJ zHfH%~Bs8gyql&kpc0XSiQyg_Py<3utdvn4}u>XIu|#nE{9r1^b_f2M%wUm&NC}jd^sG}x`po= zSURB9?3=ZTR%8ny0F>4Oi;sLtnNn8WW3nC7Zz9yJ#ut_H^Avu`V&F z0}Q=FFLowkPBn(^_&gM>7uM1Gw0NSg#I8EwdHQ>ooOkS=7x^^NO(pp1)$i+N(_*2l zUt+CdGT%x#-1P3(vHnr-rQBHb(ZodDuB4}VP>y+J1=$ZMv2V#juL90Ok6}wV&c`Wp zuD&nU>_2t3n|tFKbLVB-Qa3l@WSL1kjW(_Ry1HUf>XyEuy``f{z6Rw-o1z?cTM471 zk^ql}KHN;5nbC6_xuvb}*hm`A@;6+mR3_xIu!d+HP6vc3k78jaJ%t0g^`4($vW)j6 zKfW#ceTI33%bV>%sWyMdt{OOoChu@{cr;LBSJTp6I&*Kj*Of~Iji+<6C5%HMj1*Zb zb*7KfREE}T2$idwE8;R=*i$8GD-e16kvT)_v$YNhz4>kjm$+Vz%o$g!?AyP$oc=+5 z9lVO=4SxJtW5PfqxxqK&w2Qljr_5&!Ss7{RZ&7^yGSA$Rr@u|Tp_#QvJu^5S@)^P< z;^-}=+*gwh5__9HnR7%U=%IXLW|f!gQ<(RagszZsZh!fxc>P1^G<;+c))408bUMHG z5FF*b?Q|muo_tL|a1Mx4|KW_co%AF;fZ)>{VsHQ2*D~gWr$HrzI?+evq21|ghMlZ>mo_6`s?wULZC`Z4&S8e@g1TOLZW<%2Pm0ACgMZsR-HHP zt(*CnBCMQR^|v{R9*mxLAeFQoNYzm=DRXHK8<rPDlc>=nnt7e*LJO`|YgzSQv{g2Z# zejivZc4%-bkVsXq39_|4Z((bup4`Ht`c@BI2MDlAG!;lpHeL27vsFsMEBU~D!sxK& zZDkJZ5Rr!ugZZJyvY%dc`qZ~?5)s2`#^uNqo_y=p=*7#JE^kr7|C;pWTNYG_O?>F&2^oVLPP1Qj2S&t zaXDYB(e^WN0`fsqk*cIjcYhiM)Rqrg5`~(=Y!|S@yhFi~(m>*{OKN8NGO-W%E(Pf0D!Z1Xl!>k&S5rC!yAK^W=jGFd7YF4GLz<!EapdC=mfJb?szrr;<_|n|M}yX`8QLZkc3q%2AS1{Ogud5l29$_o6oMagckWA?Lx&>i7uu0aF6>KGYT9 z+Q}ylDBDdfGn~<()z++vVy{u_%gP^b!Vr6K=o(Rf{&biw;HainA;lt~OdlG1^A$)BQP1|?%}?xyr%M=lvo&pA|$^~207 z`vVY9)4L^bB4?bsavujtR(&q4#Be<%?zc9q;1mhEjPDr9arHQO%_91ULyuz}R zu4eJAOy?gzPIcmC(tUZBs(@!=buW={;Lp7^Q13khBWn{1$5o|JDxQVQ&b4@5yoFhx z8@Q>9b)Q=9+#!#PRXP&RadFwXmKjQWYT5Y;>D`#=a0V^xt6;PF-E@{BqL{)lMd{o! zb+6lGP&anAY+C3;$W+w=)}k*f|C+sHSz3-yXlJfQ3Csv!Cp)g={-SY&XfPA_EG~Hxd>A-Z2>Y+~Haf;;kw64OTp)_S`sSI`?AiEb%30b~^d1_gRWwbzLG7CBm`b zH@=KEKf^Rd!#VA)&RQbRYL$(2%8e2)2M8 zbMAcM8>@sn#~adr1DY)P6=Y^l8calE|Lb$$%VXW_k5n;Ldh-7rllh|Ofd9{LcT4Ly z3aS*H&z|wB!n>yGe&~kegst266pwNm@=k8GDQ)mwObd-*&zmZKg% z2a%kt^a7;l*6YG1Q!m?766 zF1V)<2uwg-9$51VDWR789W+YVjHRy}S$LmMjz$wYKNZQJz6Y z=G!mitqLCkR{_Z!6blCj;hRtJBRu@_o)F9B1>E0XEu6!u`CsqC7ysn%f2yIbzW$vR z(cf+YDPZ5f-t?=@Hgw;leE3BVS`Hug;Vi&|2oCA`3%?k>2Sn@FAKVmQqW|pKJSOwM dzx?yjNu$irXn6@;q-200%rE{V?XlXs{{x<}$D9BF diff --git a/docs/media/idempotent_sequence_exception.png b/docs/media/idempotent_sequence_exception.png deleted file mode 100644 index 4cf065993dd0ca3818941e1e80af17f4f7ed56f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46647 zcmce;WmHvN_dcwkba%IibazOHfPgegBMs6iNF6$*yHh|~N#W2U-3UmBbT{t;@B4Y~ z=T~FAU;bYlj&t_eYp=OxT-P<{CRj;P8Wo8M>A`~ssIoE=Di0n!5`FOC;Sd50_yp+< ztLcLWA0Eg`h^o5kZl%Gy;&c#qvQP$K#0(;GWu4AhKCeHbYC4@udA#sa`C%S@G&7T< zWDymeuO?1T$ikPeU1|Nee50ep|ucxP;FVkmi-+Riq z&*na-YzjT?{3c7q%G|mBghq6MQKHHy)d!IcMx7I-Iioc{l{q{Urv@&IB&Y-O*7|Q6mK}Geo?F85y_6OLZ0&7B&|sf)zs|zkMOd zNr;m8P=)cM*ba`iotL*9GBs?t9ziN_vOVLnK1kUoVLFiB!RdH=^Zxi zLoZ~gzG(%$9piafLfUe>R(?R>(p9JeRWuC^@C?aO%w6H0c-sq3u(=K-tCQJ znE0esWj;bHFCK_S%K!f2sK#<)6e^09hC}XS$0o%>8T`PMGFfd2MLx*xods%DFHC+G zlTPi{OdSgi4GklsybuY^knBDt0lv`@Rt&;BylgG5{%`&S@M~kNtqO2(@bE@x+F$Kg zE{=b@Y)#q$a|{kD{cl?-G~k6NjQAf`&V($G81t%lKI-fsX!56lNewVl59eT|?DQdm zI7BVT{159}5cG>esQ(9hVZDaFuCKOS>Ak()gBDC@=d$;U!aen(X7#d$LePb;9gD|z z2b-^*`I-?l$~^O!LZaE+5oI5R2_olNap?DFa^7t(@QX#_;9%NF0lMXUvuLUIiUFwr zxBKaA^Gw}<`_a0GvcXJkq&)4&U>4~z{pAT$CyLWrf8F#rRu|YIi4bPcnJ5lreYGT_ zg;u4pOxm>gZx^#P{-y|)HXqmwvrt3hJI8z|RE4qion_5{DyG)td!YBtd0wq{;|w%V z`t#Q`fi=s}mWIS0RuZK?y&C<_(vo#0g#*{N z=V+lam#bphbgngrU6?~u>kzh25V?f%P_|nQc#Kh2e_mMkx0IS=#d+;7(_`NL@Xcha zj$0bKxv=EnRxQ;ww^p|2vF_KWyLW}4CdkH29WHJ1hYwbI?~f}Sg}uGD3NpqOQzv(q zd+@O&1;|UaIB;>x-prI+b1*w^443IPY4@cIY3tnETl(cA6(}pPEALUCSTqY4ir4DR z)!}yz)7tk}`vQa<7p)vNuU)KHT&q1!o1p{il_uCVlIqsm)8?}cJOXx9nmWduBdaJ7 z_WQ%ov`B37z&D8BKR#}D-AA@A8pg5~zBrj#nN;eE2(Ol;Um`JncO*fM9pUUs#&zKc4r(=xFm-Yu+?fj#l;xN(IN?H}4F$)Q?YM&5Ep* zF_}JJhoVb6@6ItYT16IXLm_IguvHpGDhErE^U7SlW0yzf0qAE%u1n$WkCB$qkmw8K z;r;vyLSKv?tp4!nGzdg5SLLD#|8oE*Tz!7wP3O|HA>{*bOk zH=GRdQTza}=ehCKWw*g)ZsSe8W0s(seYK^Ao)f8)Ot*WVq~*ALlikv{Oi}Al{DBtx zmD}YI+vX{32E&j-r7Qxu742R^`RRBja&9yI^jOvuE=xRrH~BMo6j#`5(egd&(3N=&%{w#i)v-BBmSQp|6rAKwl9dJD=4f`e| zO$4VLs3c18OQXf8O9}y3Lt{cbP`w-Jz~Z3QZzhXYIB4GWN}lTR{=q?ecDLfs+1WiP ztp-jO0%-(Q5D5***{{XLY){Q85VxGx+SoU8aUnUd-SmSG3oTzcqWZDGXhS1~yguxY zL=0m+IS}#olK)DreV=C6lLDwp%?a$t;5dRc{U_+HZ>D*#2K-cz z4sVaF$iyF8Jhy%clm8IQmq-jo-Sd>I3%Dqtsk~n5Q#{rk@=AS6r#veF1 z--AZ?G9ir|0>grA>|Ol&vwmX_6^zMs!AtMnc&1m)o_xw$w>FCy&r^il4swok?#prD zq*)^NQ(uMn6oYC>?U8$>qFKDDUc@tI_NIGZT9@oG3=qhSSP+=Ox}I!l6_%2i zD?o$v|?5KpbYEk zrpw4I6O(dVC;K4nwuo@D(>4@6bqiVsASW8QA)kv)LyvuUc&&`(^tnu)0TNEPb*22p zqA;H3k7SbtH&Ev7aks-q1)Tf1l%cswBvSgh!l;M8S`EzLt~8sECNp znJ&t}SjlNZD*$99$XSQXSptSF5(zmVDU2Ek&n_WjLqb}>xem&(4A#+$dpexAH*@-L zZ+M<-NhhbzyTnZqVF%Sik^Hf@)J{%kl+D=A@UK13sz z!!x%QRxwZPOByj^V`XjV%~v31)EJ_T08T^j$7v`eKtfvG*zj2|Rx4@zqCvmF*|$FQ zMj^hWbN~Fn#Nqo_hs%;pGp@cYafZF`7L`{exM?^^s9Y$NQqMt&CcK{NUHln3Tx-L1 zdv$*1YU%xJxrDVwBzl-kSa7(-n|L3q_GsgM+)I^Oen0DdQ>P|J-3@Os7!!ka}mb4B2 z1S8^;??1}A_(pbd6G5V~$38B(I6@A3b@3ao+V58v{KCd`O>m_z)4y^z*D176UNGmA z>_?($D)Z?KnYe&}Z9;)sU~_?D>uFAL-qL>k=y)B`@lua+{niJu<0Tqe7d1puf{Dh9 z?Ow%rv2XA8L~ym5oIS6b%_-EfOaBTrD81<5{LsklaP@=ZSJ(Ye<(&FV)#h5}@doqn z_f)~K$e{gvDwVHl?(1P`t*ZGnqIRA^J5%IIMqQx-3Ap`o(f0J%L|NG#BnA*|dWaQ( z&L<%hGPzH)xa2!RI+PWz+NC7?T|GP&CRCk9iqCBGu4W}uG=F|xfYlPcgj$XLD>F(H zt+x6>6PZ0d8{-NjRP(Qct1Z6AD{wx$ku`OK`{;bSlO>-tY=9SYI*fo?<$a5#1^*sW zU^Ue?J(j1Xu%kxw_Qvy&>A2QCd+qd%N%6xWJ~hQ8b_B&Bp)J>&tFOziz8f~B)P~)$ z0Lplx)deW^;w=NH_Mxd&(=q<^OB3)82tKHi(vn8(ZvNo5lzqCCQ8Ho&T!a8#H<6m!rFVDmT8I? z;+ShdrROyg-CI(|`BCH@IBtWx+>S_t-*Je3d1_0ISHm$GfLzYJ-I=rYfuggmM%G3^4I?jp7bPGU_pSdd1IUV|V+`0Ur% zM;dCQhJTxdcG!N*99T;}9)7fWU#ck>`?7Vqri7);J1#P2RB>?n6w0W5gm)mOr?=bu zUMDM>&8YtIE;2;FdYxA+q8uJmjl60gz&Ocs(l(o3M{X`c&w|b<22cMZF zXxM%wd1f2ZX}^#yw7E ziFTCA#N;Q@QVL)!QZv;z8W?;2!&N?HgZ#MrDaS z1AUJZMx5QiBITSu0Gjw+yo5~qKdBh;UcJ{yvUq;4h%+iA z4z2<3@>sy0|pk9WoL5fxL*k3B}MgH17nb*i8}Wc=n2 zHqzBg@VN9V%ROc+a}m{z9u2xpy-Ms_Rd2V3a;CzG;q&DQ{qtj0)<#|yto}%W4M9li z>Q6h&5InB{Y3uCac(a4H`QJ3vYB00*O!vDE*C(RpX%5o}x>xzESnq9ZS*{EXna(Nn z!LhI!&k1>OvuJacyY^Gk1{!tg3R%kw_`Tn2WtAc7lj%MKl^&!U*Rii?X2F0~(tiI< zn}>Z!`A7v&me08IW>Ll;pQ6X+fgIAV?z|gu(zrTa`n#LQRSxjMFDBooli2h8Uf-o^ z_obg-w<||)?$dm>L-Fgn9ifja25{5dMiqaDh9Z%ax%X|;#HP^}s#&TZisBa4pzL4D z3f=mNHy>v9n3f7YThw`IzQ2A0giPg@qg_sA;EOc?=e!>4ILdgGRT6(hF|jrviVfTu@WCfDf+uI% zi{FFG>6#1A>fs7_f$oxer!4vWrJiX9fXX2o3X6)3g;i-g-wZ@8poG0uLwei*A#JxY zG=;Xcjfy{y!7&Z`_T?K86SIJrXw^!X7)8_W1K|ToS(bR9^Ulw&Ks@u*(8%Zg{A!YG zi^3UOz+-!w>lh=7=Ed=59Evv`}SN|W~?umS;Z zplF6X6;*XMvnuEhWmWFn?D1dPn;|YxA!wM;IOR1pC#R}R_pb#V*9V*5UpnXhW63*x z@PA$MXT1~heBa^PK!)&pBE3;+#GY0b=W=q^hu&k45fPn{{@ySYItDfz+aC7(l1A2Z zI`Ga?`l(uUJ#4=?6aC-8(bAaPWg6_IHG1C)H>XaNy@A$oa#EhJgkB@7#C7O7iHiBA zwUrsbSENi%0ok%B@B_`WU_zf0*_g9x^gE&R8*i0TjL|o9R>Z%pK||{z)(@9LzgS-X z0R#5i8I3J8>e7eq?(V_C!QT_*CCG~?+FqdSPp9Nwoz8n(;Q!er9QC<*BDHh`3BRMv zwgM}epsPN6QZlD`Hgga$P^H6(d73>=%jzj)r>f0IE_YkXPq6L{VMr5`0qIHR0+7gj zZ0->?CFh+t5o-s9g{dYyd8SeJdjIq^gUd1=9OC^CpwGeM_z$vjnD+Oj@*Vz2k@eJB z2Ffx}floIIGU(GhPPXhE90a_sz6{IWoBi&D?VdMUCJMOhoSmJ?C$USuiCRw;bi-=k zHtxoVYjHo`boxE6#)o*n6*dI8VODB3>wr;4eCY9-@%i6 z|5%Aej;fRTLP3U??_o)KGghe5v!%{(29?~P(9jnkbUjXYNb&G!87#T_+5(U(E_miW zsf8Yp(!HtY)b9TH6vzW3agy{v`w6hnDA!fvFD)x8IIOds-V--V8{&bzwebdH>BLM2KhAqyDsW3?J9a&# z^v_kjcN0G!&anOj(<073pxZ|Wx%?hya#-!dWl$$!HIA6UIo za@?^d4;PxF)7|;5wXR6=1STCk82zL5q3+oNg_L+oafsDa<@ND|UUsD%G*e4MLu2e~ zC?UsNiVbp3GYsu@pxbW_r1HJe5=D%Kb1yua)Sfk1gF>O8_#$GmyX^dYDSV#my1(e< z<)!lB{n3!L=KIOd29pg^;Y3*j8^d`H;MBZt1m!5h;ZO)ZAjQ$EB^xg+Km!dsL~hPt zZ%*etc2~HA3DbZhQ$EAJcl;ZRHks_Gf9 zNC*joOD6!YcHb%o=K^e7s(E#NI$y!-b>&(?#~D^oum(CQvFT4<#0N?0OvFe^NIWm7 zu^5wEaj|y;#TT<7Dn>FPcU49_7Tupe zelQ6NzGg%U!DHH7SlBzQ9kLLRl6)p-UV%FAs# zLU6})*Q88A>X4BOxX%^27^ocP!!#R+$jGlO5_Rfq+}z!{ZGLJp#)GmY5sX#I9s&ac zvz(m?xS!d=&#w;3EL`&7Y!222cGuPl>LVYckO@v2qdLl~VZUzin#FlzH+APIGUr5a)fQvDKYpMd zP}9&nl5|+>kCWWl*#RzZuRNG7iL5H%uqr?L0`(OrWz#j5DW7JaP)T&_*f!PB^Mf>@ zJoe~ihOh#7vXOj+ysm{!XMS|IkJQp=VpG(Z;Pa3>$y+E|QdHougCI3^Sek+V>qKJ3% zuz4VGEryqNysu{!*MO%7_vZwLY?cSrKro+%LeR;}4g~o4L4^=gp}#Lapn6EHi4z-T z835;u2ks*lfOLLwL4U3P^NU?!qX86dgU@Lb$6wA54z)e|ApM4z#|DQ^i9)Sdb*9cX z;kaD43CTPf2(~)fVjY87;>vn+`Md9&1ky6tjNmS~4cZ>%y;@X)rV6<9<6N(jw1QQB z&DU5a^zJUNgBph*TlA->fl8IuE(ZMo;wy!*aPGJL>CSAU+u_Ce`Hu_{a)naeraPkw z3k$d0T$B`My4Tv9nVCh9@Tn9imNw3sfN+fx>IlWxtFgenO5w33W7Yp)B^tncNm+gt zGF(w#+As3{q8cRDz0=JxOAuR{E_01;)KXy>d1zc=O45@FCg0;pdEY$zz-a(e9wc=i!!GDPMug$uy=JhAR;!_&HpTNKEL#;?0ntw(pv@p zuL}SCV1yO1Yvp^wSQ)`xei90Q&Sml=`51~WqI&LBW)lZYIM?K%u|3u`?|sXpD}4wO z1*mA^9nbVWe55+J?mf_d)iK*Kt*6;|+|tET_M7_NK0PoNiMf=ZTM9gHvEpIpLe+q9 ziBeWulgH`k<7d(Hv@CvMKGrWzr-PLp+YI$c4-sw87m$ZrZf{CXXrBQYrogqT z;NzZH=NHwVTUypRmL|ITQg@{@h7n*R#c)Al*qJRgSZ^0x^Y6%g_cb*v%!d<&d|D>v zQ65;pu>@;N#k|CpY3%Xae*h_zTmu_Y;b4@5;pAC>#fHs-%_*-%lzJ2K4QbU@!pq{& zxX37(SsS1U4VYr7#P=B;U=i5k>>t$ToYa%2+jGsw*G2j_uGnI!1OSEw*|A1J-0aBS zZl?=$^$*?M4>sS@DU=ue&^l*95FNlQ<+~`8YI{xu`A98`)!tyJD~mtT+scevcTu<- zXz(AHDU~Si!_<}*DcD%;&1?4m08H&}(J*g6N{@l3+-f2J2lfI*^={)>*w6H9^E?!s zrV`cufxHN0K)=BZ;bB#SrbkIa_@4m;YH?KY)Ck}f)!tJCe=r97C~N!Iv(stoz<*x! zf1*VHKMRMU{3qUQ77QgI{EtWfKe68bPpJ6UZvX3|VlqgppigL#fC}&mIT6vt*}jp} z!{Db>)YNQk6Xkju_?qIkKA~K}@7isDe(_PbLyF4DLUR73H2xS^y8xeJ-BL&kdY@RgS zH8lwd2}M5*2n2)4Dx!2?gT%8mIUm^-5~vrJ%$rE6``i00=?p~9BK3l0wN-$ zhpc7*Ec>i!-t{YVDT zFC-?<>wf%}*^=HtMb@FK)oEiY|t9^F%8kFBnm>C*r)K__OxCk0G z6NT$TtQEZPDron)P;kD4hJFXA!8pJN7Jjua=@8gp7=%@O_npCPJr95IbbkcBpc3Z6 zci^eO;{)m^A`U@8CF10D+Kk;BNEd2&x6o$x$lG$Pcr0HbW&O9L5%}fI0p9ED>rgzV3!eZ@OE4`l2z=sg)nML>1j~uCR)AC#szsbORn<&J zNd#R#zeYhrD^o4PI>C9b`&r-TNW|-kpri%hC_qedq$B$iSjg)`a5DtmsOTF{rY)PB zo6E||Y8VSo6njiF-m3#x7e>ed2L~s1^18`gP0OO@(3jl%szR^DOQ{>oShNnci=G1q zH+e=6pvlEhjx;PB%5%ZKGy#n>P(6PGkZ+)q#9`VQ-Nua2GRj(eLf3oZ1BI7>ao5b1 z!9*!)41YvU;gJk_Mo(XoIaV5Wgn&|yFKN^j(NTYwovl!xSA283I@_lsChGKZogv5g z?7Te<>W@(iy=viSuWR=>$V1-o-dAibR|m-SnfXV+#o5>ajqx<2W@C3>bQsQ+mGPqm zY=$`qEv6KBD&^_L9uys)Nn5p0iE{Y|DD|R4fXiq%xjS+Cwt$$ca}eNCceJw;C4-=Y zswvSM)B&v9i_{GnQUu2drj8;9WUz59TORZ@Wn}1q;y%|O zh!T9VgnRWV^f{NCt1AVi6gfqBRGcA0#RzKx^PApeE`}J}BWK`cW(AHL!Znoa3 z;@D-*O}Z>p99H=uQ?h`IjiWw@qj!dam{^m)?Up(heno&&x0f7s`66A5_8Nc*PfyRx zsn2mAK0I)Rpc2#0fMy`J<70PTqfV4US{@#rES>ge4KDNVHEQ89$v>M@`cxBj8r&Th zi9m%PT|iBr|<+&uh0$vR`vI{>*n!7-OGl2qxO!D7}bhn8;@21&v_yjyLWI9 zgWiWrln)BLGg(#x5OYRF1S zF~aW{M6}ru2T)FA7VqodQG3A`UCAr_pJ)JpON=ug@jKeCdvE;~&k z@RU>ltT02xpjQCi_43n4kc&j{oAkz>11>mbvOo0FPHhq_mnRK+U5^F9Y)dq5Hs>jH zeUufX7C>H{Hb)FnHhSE6~R*jWIAuUvKsIBXkr z`Zosln|ngBWZWBzNcOc*8p`;l9%ma+BRM4Q;OlH^lcdI*!_+kXrClhz zKdE)#72V2U!$|msjh8MM*2!^tUL0f+#G#@QbE7P-mABkrh0>$uJZ)5?gEt0nx{wcm zXp_g=HO3g#dzqC-qh)W1wPl#GnB4P4L3O;Bzyi|GwkFESE2&m+SzTYTE0sEWov$RkqbkJ#h6=iko_VOnUi`Jbp4)crwNnVi&UGrg z3DPEAhFJbED{w97os*da(1k;WdO&r_aXNZJ%I}DQJ6&rX+Nfn*A*OhzA>jn?k5akY z9HX59>9kW7DhTL=Zj-wil0~ui?ajedvob$Fxd|cv5jN9ku^O^3H~~;LXz_D&cZRyi zy+1k1k?cf6VV`0citFYhm;^D2F=Yk<`pfZ*vkzowh4>dk*^H<7qJUog)U3@JW_529 z=so#>U;?BhW-X51WC#$dK3W9rp*q>wEikZbXPN>rFsdC)1!BBe4<6uiSfyYPb1!BN z3PD0Zc%J-3*u&Lzxi85<&2f2(SZG@R*EgTSociyCvc##-Ky>_2elGdgC-_#ApDF1Z zzJ3i3^?qcxy9SP08GfqZs8D6$t~%pfH-`ede6?GG6r7H%72(zkfS_0BVXmz~rvw_X=V8}Mi(7^jeDNC%9&=Idzxj-y$H^xkfk99R zqY8^cV($}C(W?S=-NAB+T=fzwkr|}EL^ko1tPmaHw*%=+c*m9mtMashQ#Q@$NQ3GYQxCPMX#NZNw2!!qGR{D28KTx{jDC%4z987(a$!$%*V zzD&k3MbFlqgJ}k`isZ4zolzVu2@s{wvs427C z3e37!)R7wabiBakB_)<%vNo0KV37} zrRQ~S0xFYCRxacYNinE@3i7ww`_&3cfTZ&mAUgsi9jZd}t~(FFbifeYwk9hy6r`np zb`6(kl-sskJKx@11M#MFu~0e3_^}YtTM+7L{7%y#Cjj4C9mv>SU8S|14h|{sqWUd4 zO^=&)GJn1(+6l%Vbw2C20wIM>X)!-q(=73O0?Xy)rH6;d%XGo>%m8qN-wU2fH#Hg~ z(~*nPVkAfcoxuhQLDj_63H)^q3URih@s)Bs{BgD-kQ!KH4uX5Au$B>h*Z?XZ<_0si zEfS=5XPx21Pjx&))AT>MEYPqmi2ZX0z!@ z`wbsSYR+*peUmZz80amj;N=9GEHGwr0P-GC+o)_%Cwx;;lCn2VwtaylpCjWWnymI2 z_wuO@2~3u+HD$RPej`o9wZCBx7`-`QCu0KQy~eezFtecB!K0>nFcMq;9A3!|gfyN~ zSZzejqo&IF=F*_ydSMF$eQ04iHxC;=UoOT8z?dPPMdsgTgMyd)@;K>?$;C%S*3tQAgr&< zaKB=Wz7?9+sYT}OGu0xp+DDsCEIx4gA$Ps9|9A^kgD2oA z1DL7;o?Tt-Djk!xHCz6l79=+op-Or}<{;|C3@K0@0zr=t4o+maH^Y-CTR6wcV1YsPEk>tT2@vodd^+1 z7GRoV`NWM(>S|(jczSjkPu1D{2Q!e*WEO!Cb>AqATJHA~VYWRkZR#lvQDxC#s_U|P zahkm8MUZ7cp^RgxBi$DcnubvlkUO|@*DLKwHgzH2LNFt<`ilMbqlemhy#CCmbf{+Dv=Xv4^H@DZo%s*A}`O?*Qe+g&-~_M)@8mZ z6l}=g@7}A-!HQX+i!T#?qPw`+oIU;AfLxt@`Nix~6_cAWH$Q zEb46KqSZ)xI;_qCsBm{KcPD_{$vYK-4VTZihacYG2q+go8=)b(|0%G*-7n4VK14wS z-vhNK`>sLD#(Vd@7(y`IzgMsMvwHtszxB@pk^26}N8m1h8m(?%BmM*=|7sPZ|Gn6s zzU&%h1QG4x!V(DgAv4b8@v^Opo2|Z@#3$JUPuTut<9=}xuCRB*-~uKACCleWUuio3 zy+9B?>S5O+v&P9Oa5B!k>x#nAC^q=tC?UdW?I=La7jB6EqXW8uD;*BId@wPW%s7b@ z%o7P&MM!LBc_Y|6fhu}u3aJl75UHFeQHZF1Ed{p|Xyna>lR&zv%F4N%8K*W((kSa0qjBGfeVUkIG^HXXS-0mt8)uv^MgOyJl5lAkQ>e#lvcekv%RPw*ru_ z0k;1zyrBebe;*|-4xcVCp|KI{kz&VVMf%=ss#1TV>`5{Y%de#+pAPq9ED5Wwp-*IA zmJI=^t@!di?)rLbk|&&KDuE5S?4KzCvOq2kNKj*M zg8ivMU-a5?0#ZyQeZcSS^*+{SevXKEIL82-)alVi&C{1ageXx-ZcF0R^`dH z&Zju46nULL1s-|czMt4_{vC#1pm;1M0FUOV${#S>$VX!N=8OIu<`#?8+tt1aW<4?* z1_bf7idXAbwbrZsB}B?&;7zV*-op>^28;F2@*oxUSx;YfX7>kU3&oWh5F{{~2h|MT z4v5(1%%-xEXnp#sLJHROKt`M)Q%YUPp|t z<~=c5zfKPiQ4sbH_xB6c80zT2C^j(5g-AJ9%;@vL8E1Q+VIk z*D=o4l$4g2n{JNUJVt&ilQIq@uGf6_XOk1GRd2^cZ7cB9MV;rG-z%GtQ`mlvLF;p_@A9bJ7{zn>13OlohsDXY&RjL=q?XeV|y04v*4~`NptT zfO>_~FN|%;5Mz5Mr&c{rmS4BqbsD{Ve962U_|L}zq6Fz zG~6!tOfGhCQ161_@>b18M{|k;#b-WcuL*risdXf?NeuV`4pvjtqHv~T9)5-fHe|N_ zE?>jGJ(Q}lo=LuSBY%bJCrGfdb#eS7YkX&plrmKt)sO%DuwcHbfb;~79hU4!NfiN= z6yf1Vfx2=6RFwW|b5WTu+m3UaPh`<}a>s~C1@Jm3x^psx1ajGdgfz`9NtPkxp}ijO zeM9JLAc}>KE@57-n+G%{_ruk9B+^^IN_C(e^dK9GzYc!U@@|V>y)VKq^u|&^e@vBV zb+UqN-85fb7o9vg^3w^yRPd8eFqi>obTfh97<9(!3J7m33FiRZ6HvObsV*!{NU`;>l^0+Z4aB)F)RxPf1^pIqhMGG=<6Ojil~j)? ziGF^m2%sAyqrR*dEqG}j+xyBBkQ~C$P&_V-P=(}W>33AB?NpEM!V`lYSkL1}@>hSk zz*4OdoOG=!U90b#J}JCISF=9tD*PX75K*gEy~)df5)XPdDcsi6b+&pkGBRdn>(`g3 zMfYz^V};X4@l`rx3Cp)?_8i_#>hI8#~N9i&SbQ!nVD#>4nRcA&N{w&o&w6%44 zrVWOn!WUu~+9;Aj@CO*>I+__+{KfZDaNK(~eJ;1_gx9%qF(F`b63vV{h{afj0O&WEd^sxK!;i$rM;2_c=B@!n3~ z2eh$^-hqTMBy8Uu6e%pQgeGVtqc?6fd+xo40$@N)ibvy=JW>Lha#^>L&0IZvBaUJUA z2(9Vr_F`-W?-)f<2M&CEYJ`b}!P=|i0^T|Bd@rQN;JmJ+8f$U-CKBR9Dl}aq_SgX> z=RJyI@0WPSv!<^~ETEwT+LT|Mw~7A3$bfZ|9f*jFH?}KG8<-LNwp_T*UnooG zFd)Y7p^#Ovo=L2vHY_oy)s$g#x02fhT|O;mn8kxU^T_CHO<#rd*VwquJ5~+vvd+_) zSG3YV-7+4qIs9UF`lbMa>HO=tG-gjNJq`XxqPHO2OBjP1486`40zqT;FLX+c$QAab z$Y{3S>{g&$ovEOty6xBc17_qr zr{C%@dEJmaFu4RV?prsJhKr?(Ce%?DCfQmaDyu}tWU-|MqcC4SQLvA?PO$E3+dNSz zZop+AngPJN9npdQCy2+qkp)P#yZB80K{HON=vf@Y>Ee#F!8CV`TjP6Fhm~FonH<9o zc9Hi)+veXdnDRl3kFk|mm*m42>xnX%smhxJel~+e?qb_!w;C3?k(~xPsHV_6IgdU4 z*o+L*g*L(($)4_92qGF91pAgw_r-BCntSA$tArUPX&xw1D`=Ob&qD=8cWk;NwF-ItW# zLU4L=JzW!^sOHC=5Y!(X;aiw2ytL?CPI*nDMIu5&1m zE;f3~A*y`?0~`uOE@rqg2smTKaXN0yk9*vwDpx&3hVze~tB1TUeeQ#jEt7&2y7rLM zQvG=|3g%R;lIS2-w`jm}6pJrT*})jyjF>y zKGLdkCl~(N5&M9Mw(UT!7wd^`qW}<}NI)a=M*>R|2}ucLOqov_jeHcJWx?(XK)Zlgbj6_Ubc{;VjFCK zJKj`7B3lP~j7nDnQ@$;s9XEt>!e(GL5^P=iEouP;`=C-A6iz1u7Cj-uvz{pIt` z7rVHp3dLFOyp^m7F(8eFk?TD!IvLIjZy#}@X4GO#o^GgXa&{<&cix_xwOMT6H0o59 zY7VMCoA=feY*)u=I)95B8sF(AI`T5bFz$2?x!%?8LoNEtj0nlcN6!7J z^@}UN3o|`ecUOO?b7$E@0#G}(0XocG18%rSB3Rf=wgEyKp(hZP@`V(4d2uFZ7|nqd z$}wOQ9UnbG#cA+MU`Cao-fet^%5HSLneVRK1CtjLu9*J%)z8nl)`y@R?9s6d`-%C% zdps2iM00{j3vaz3hyY{EqP&%_3gF(KuMn>{?%B>xBgJ{&zPV0>H=nGpd!vdj&ZJr-vL*$jh$DlCq&c-F-evNl?zJDel*Zk>-mIDe3$Ss095tG%k^0GVv@ z;oX>z8{mTCFGyx8pjR;1W={b}ee0P6R70}+d2o1(L}Lq3nUi=cE;_@RnlFl>p!_s9 zrzf$yy=lJMti>i`A|wRqeE)uI9dKltSC+H&N}XZ4w8+B3WSZ4ZF7W^Lj>|T^YoG6w zt(frt#=e*1tvK&Rl&}X}C_s#|2hVahWriXXL9}|g+dx169a9gFi2vQ~4cf`Pvmu7Z zfiYt}{rtzm5yiC4fSNWasR(LPJR_rSGeSjZo-5Cvl->SexzONKv#*lLG*1RE=k^So zk?|y6?O7K&_q;4tWd3`FTryC95ytBspK-%8;4PVRRvnJx=Omrvv8gKz;J0RVFvl;> z-f2Q$;0RWw7&6rCGndt&43$q*LM?>-@oZzPL|mtaetB~f z=H>N;n=#Va zRnz0!(!zsQzv%`$*|7QpIahXxd2_jUrKI4{?J!@wes|FjYoFrfV6doqZDha8wlM^{-Ry%_qSL%`9@jZN;!i?^V| zBwSpI3@tNw!|9tZA#iz;G?P$aY|3MFBpHL3xpy!0cPjN6LXs2Pwj+RJv=8DblPmJkCG}f@Vfcx(c)#Ilx5hl zwcD&i3ikxVVEYEuIBJh#3RIkHkiMnEi({G7o~n2!J+t5fFI~q)4O5m%Sl%Vd^LZUx zwYWTbAemhpJqu5puFS*5>SBHqlC%1&wOo;;!!cH3F8ED<@))s}uyd8kci%k<<#_f7 zmk8+gM}P{H>dxXQP`YtF;|#lIF6tY1HS9nkRG}{FSj|M)*Sf;@-qJiAk9+x}-rfXZ zIZ5zMt%0}$imcC)CH5CKVPP#R$YBB*oLdf4l8xK z`LZarPCM6ZF>rJB$AsEwIh&KHM=@ud-E3$1Gx?}(seMu3(`nv-&Q)8om#+{#sO4UB zLQj*Q$90IjA%Fkq(M?M0iCV(cj@+%IZV61?$osbPK=TsFxBE&A$=jaNAv z1&=N^pmiTidbB-cMe)&lPMwV^I!)dP0jz6$ss%+-krxj=#gmJoq-W_BEMg~L8TMTJwb;+ZAGg`cktm1N+)Iw4(b}Xh-yGU^ zBM3$Qz3W0`d$`DNd0&q&K(3O<@j|R|{Ilwc25n~2w#iMuDHz2TWdK&tK+U{W(CXBV z>+-F^(exM2)ngn=u8em}mZZgeClvCQWI@DCJY8)tPJ<_<0uFYj-bWiQvD{s!2FXPy zra}3&**te=u9HNRe)v&vsr%_g&Tqt$I<)_1}!G`uWtSK_{}W^MQ0ra(SQe?y2pb=uCK&1rRV(kJXy zQb{9^$yyc+0EW;ygkKrL+jA;3gg-;jjNh7?@ha=_MfO}+`=0WsmW(G`RD5InXnLK* zlnu|kNrkQdR{BQ0IW^?Xdp`w3bq;ZetN$5 z@uA4VGXOjE+DoV`JUsN@u~Ti>Y|9m-Jc8Zn*fpuWveYSA*ov{9uLO z3oJcg{)vg(jrM`e6o~W$|GYmXDI-cHK8_yMQ5UvrOQt(g|GV~xr_{iTOr(yypHq4@ z5=njaM>e@pry3zLSU5PCols>H1;bTR=D*}e z9vt&4E7x6C=DH}e<`b^-C8l8E+%n7B%VJ}{!YzJSW`w-K~F5t9zpGENTE|+boLbSa3mE`SU@;^WG z(`Id5ow(=bw3dMBGn|LANt~&An0E?IgC-iq>ftiUi?$j_x^(%#tXpUINmQ$UYy!C&DX2fpJ6=p2X>Ad;_{!2EKncetGHB z`4#lec)W%V+N4AYaTmG|J}tmz*zm2g5t>uQ2T<*@x;-h-ea=IOHCr-VW>G?n!xIF- zL^jPbH|oGs)ALPWKN6ERGKePL4v+F%(@-7w^h-(qWSM2o`-{rNS#K;CQCFH=%h z&VK8gw}rP)2T1r=s4gf1>f%bR3!ebhWj%iigLrQH@WoDvn`e z{nqNJTkOx@r!F3Mq^zY8Iwk3s#q`IZ&=F0wG(Bx5-TD0YeGkBJ%8XqF1*@NR1eyyg z({IS0T}1VX?J@Ug6(T*_v}DYuUn$a{qkInpZjQ=mZr+9gF8n&=(P?dhkSwI746blp zdRKr!$NVP0G#ZC=z@(;J>12O?2n1%B*^TP>c}*I3nY$vKxVA@l_Z3C;Mobr2)ZTIE zwRWV6j;iqHQOsfz$iLBtYuB`=2ZhQkQpo%9RKOiH`9;O`H=RAq`-VZC_L$_23?ejQ4jG!bw+0qmTYn&n``57vL)+>fG{glE zwe|#~Kd+h>5fOZOm#3CAzX!c{CF}(V4XOiLIld z^_e6jm!+iJH}5FSj$;zIOJmm8kNfzDj=jH`>7xj}sb9;&w-cF(k~~D{IfD{fYEe*p z?Z-&)@v0gtx?VIrAFN_dAFr|-U4fzrEs@V!)rIVzR{SpWyo5SeIg)8^znfq<$zr_M z6pBmz`Ga6rEQ`7@V`Jx8$z&~GJ(M*TYd(B zo*BH^n0PH&waGT^kz$NaO-+NY>-Or#2 zR=k9JlIC~lr~_F*FAw7-n{eIAK-{Bnx@Q;UPev}G<@%_(2FcB4EX=g)>J|58%1Ld_ z@{;XRW?#h*BBZl2X>4pyvLCtsysXB>#27uqKS*J(_KH30@_1I6IlH;s(G9^O<3>>| z^XE=7_kZ5K{Kj11+0xmci0C|FBQ`{keX3l*gIc&0ob1Sw3HP6>b z@xnqcSFZZOkEW);LCPU(Uc)}a@?I_<_nX{2=Xk_VQDlXhd4A{W3A&gzXD|2H;+ekr=u%_$C;Gst70VEp3dz3~evBNe%6??dNj-vcb) z+WwBA_-y#p@I+;NvkV2r?*(%RVhJZTKu)0)Sk6f%aZc`CiYJSD1`goTW>L}Ok z^zUSse~SK;@vGtS_pA^~{^Z9ax#~-tO+GSr7WUVs3iSo#K9m({X0fHg<~?jWYh)jS z;LE?De93X+m(-q|POA^+u- zF6Ih8&Aln8Lj!Cz|Mg}Y*8SY$15O4TGY_5owfFW#+-A;OAD=!S!HL|IIK%bYXaFHK zzl{mX>+JUlHKriW%>ZF0yjl-VA$L2=xfGeXr~Q-6tDob`7Af(m@LQq&Tp4AB_mqfg z-0Y)&xfP*bFS+H?*A&D;w*V%#U30*(pm3*_m~nV1)h8kbwoH<#gQ<{`&+=T^2F7hsnT=ku^X*Y;w zl+I>)AEL=<@r+x8g$uIkZ2J7s$=C3-OYcJic_00pTopiGKQE?|ecqm{ zxL%le<0aFR;{oR~nUKuPJMZF{qeQ|>4L&$zyrbQoQ5Uz?fwH=QfIcK2U7DK>j|Ik_ zYOmKJ_+zs$(dU9Mr!|+!pszBJvfR#NUdh9qxdTAuK(m3VF`5BjTIIW+XzDD#E=Lg+ zYqJn;1x~eSuwIEpa6Ai%27wIPR95K|-NO0So;0!V`UyjMv0ZI#*{cMVRBT6NRYImAUBe z(}^)JxW?O!RjUC?{EFQGxXwPz4uiIu_z1<(SbcQNCdCywHeBr9Ou6Qj;gDt+P%qDz z6!$g5)1jL?eywuBERLt_87-Ag}#*Qf`RY zI!;uRTxp8134pM5LDzoEy2W6A~5>J%qd%Kxe1lEq|D%#ZO^1m_!Y*X^@i zD=NDCkQ94oVfbnU{T1Yt2S|AagU%^{dZYW=S+9nN`zb}lVu>K zzRp*BrAgLC?@({My6cBY{ULe|KW4%A$vhnmWHVjKNx%4>!(k2~JRAQ7!SX z(`rCt%3B~Bm^xa_x+<{sy^rhtSnHt--h!Ryz0P0oo%@`?t@6@n!pjER|KW+wz1S-4mM{&S`0Gp3gS%LirMazNgB`_hq!= zK^h_9Pj-i={ta8fHO(-&)&dl7s*`Rni8^L5q#D=OD{u5H6y3cm#T8m1IdW|(aI!3!z zRR5s4R(Js18_eGBcpbwJ6)5jd&-Jm#f1NPI7SdgPcjwk{N9>iEJR}MxBF^CSVkv{m z%S^#ikM9eBA$-^4%F*#3Zj{JUppgN6MFAC0rkyiY16>g<79~QCPSdQ&w50_jC37;#e)647GH0Xb z#cx0~ncC@0RJ|Qu=>6^W#-s3JQ@tV{{bb(!c1)T99=Wqm`)A)p2sT;61tMw0DtjVG z-m>;Z(5n*0u!Xx4y=BW(B+3q_rO*6GBc(wS^Fu|lpid?HayS~HG}E_v^Sjq&7*Q;}stt zy$TkkQd^iHwJn)YjC|QHL9w97%t;n6 zZG7u?Ou^lsw#U069~?07eRSA&*py5<<5?QY9J#ZCp9Z*zHUw{tSbtq1x4DDpV6d^` zwg#>Z&6Cbm^DTLVw_!x0H=y0DOuUM{oknC@D)=oU`2Q6ZNc-F$jY9al@-XJ7en z-?=VPUDsyTu9*F*%6q%Pdy4n=#t(vrs*<0B#PMkcMIZNa72TESJ~elt z90H`O;^OB+dPUx3IV*Zm@h6P7H0o&w226K$Z`aKSftthM>F;ZR+E50br(Fe2D{ z1J+_g&?PER24PZlN|XbE&Kqe5SLuOqhl-=N>8RjXx{XV~thB>EVWDH(BFv(|pqohM zu9|JfX~}V(bEY%WM!C}#UK7vgbo=#hN5j+Q!42hsya&{jGN#cK7(@@D0aZEa3-n)S zg{@-Y3D|q{p4@quX-X(eKxJ?ziE4AtK8&zQCx4Zvt4TZe#kFE`5<=pW{U@74ce~9i zYz7}zJ)_ldukQI(YX9yu_jMy$x5uPzV#@~y`!un~uZ(K?jOz~_tvM5-2QuX>wYG-fKVhs>naL=K(F!%u#Jl5n1iZISsmnJ_en3EvpLcKm%RM3I>?jw!TMyMP z?gqWPqn(xb!>@^~SbArkV3^6Jt~y`@pliB|9HhH4c`I&vmsy75EvMHfRWDQ(y*=qGujtR{HxZ5X=4O4Co5N2T=}6D|xSdPGXAo`hM?s zxp|WmeB0J91RNjJ4)jk+7b0Yg4;0%&Ek{hu!gcM6qwFRPgDVN*2Dn)N{DDI(cTgK2BiEn!zj+Eb6n>1#-Ew3OFis zJ-DI%Gd|*xWE^H&)1y?yFL#?xwG`IhMv`sSc4pjK3{My9>SBLFVsj*-NZ~bnDJZjE z3cX-sb(F)#gIyQ&6Ys&It(q$wxQIEXvf*5P+hj>av#{Tel z8n*Y~tM}cc8zSzjQ#q$tc)I>p1}BkUYg~+f_KJ%Kjo?$;+fgjPx;s77Po|J6dh`{_ zL_Zo%ES4;a@{hPY)^~r;M0+?IwH|0tjEUx5G_&mp`}i>gv%#l^^0)qFPibS_07h7q zH$|4&INdjYDzKus1^V`LFx|jlDR`6aalF$#-|_An{sQ`y+%ZhmkW`!Bk-hI7?297w z+VHO(V?Fy>KdZ^w!L?Sz=6ANws&|ql{tcD7Du?Qow1?3tK{T&sUY*NF<4UIbl8D8tt-3^^J*T9I%xQ0SA^6&|3I z-@Q(sYap98q%79^6j-a_PxOoJ9Gq$M(ooTPmZ`l-BzO?-ZunEXKt!d;`|;0sRtA?9 z)ui>GRq{LDx11sPUYX0c#_nxV;XEh$IgSYq!wT+OH=QxkX1GLmt!R)LEXDmZ#vn$> zP9v2_*V?0~VOR0}E>(SOh?KsYYPsXRSFqW|=_VglaMa1(Er`2;cq7+apzrtXuqW#Y z61x3s(JNL`%pdPj>lTU^-t#8IzWZT)YTWxyyg%ucogLfZOu62L*WCaa9xtSKX=KTZ zzqi>P%hyip(XYap9Q_LC)D*C~zm@tCOA`}>=>h+n_r##S_ng;Xc5#3Uh~(d3hau4f zb?6O9yl1fN4fr{A4;UV;ZnX>_ho-~qRQKL*mjg*Wp2T@9%yj3y4LV!In=1-85|kp&9nRI&^x_QSc%ATz{RO z^4pF0|3rNH^pKw}NK}kWj4Sv)W3y*~CKD@bD2#>5pV8{gGgW&n{;ON!jCOT%Z_l{F zdrwOm%gdX~`#x^7V!#$!ma?NG7u2(p{>*fyCISr|o$jZg_ii3EVNOTi{Qo>nObj7_ zAvc=OSCnj4uv8uq6;)*N3^bZ!iKAlE%`P=BKIKMZz;3@B$p%g3FcGc9e@2HF$pHw% zw#)vHAv>CzJ&TWQ4zT;HeWL~0zELKG?6{D|!p4MWY@PqBd4LM}hbezG894rr zH}Usz8Pz3?ke`nkvbWdF1-=$J?0@0_r1)RxSLk8*>)HN127J(8!~EwhuolYD&TmFo zMt>f~AmIEEN4ABGAlpLz8i?~pcpk-z3#5Ga;9uI+(4T+*{!ds4d)PUt&{RB-ukBsJ z7Z4ce)C!EkrORfKKx3oYf@U`dfvhwzIB8%nE;CJ3yn*~tWkT@$L5vrm-?UZ=0X>p? zLT3KJ(ws4Xj>6+0vJ05bf>b7czpBKW6RzI#oO<~%z4vj^7N`#W4DCv~+0W8SrlBf0 z4H{ZZDWKXi>>$sHbpmp1Zx7n};E}-RrT&L)C#XSGiP4BAE<U<((SwaV{O?j^mSvejDb>U^5fl}cV2$V2FMJbt(CKOmL+3iQ*& z>E?;>ZaL2?=GQpRe<6R;SbZPf4&pDOPSZg{HI7r#l~-!t4w0|@v&4F_>D?YZdQ{~# z*O>_Y?zUpwD15hfiA5+t5`{l>dt>^>lfMN*B4?-OZbUe}Trl+d z)JtrJ(PHqW&_?=;=GOB3#s<8}3wiSJBb8njWpg2*ZvjcJG&&iKIZy&G*jWWL^d@MWaLL?YKV?n3z9_O|kC@di({x2E6(q?ND5q{9i)2%v;L zAkO1@vcCrERSO-l7lmM1)!7WPr3G{6#}U0XrMHOz^^lBUP^OU7jb~Oh1YR-7J{0P( zuXzFI55&~WJhC&*zQ4EM$usR4dsM$A4?XNwNSK1G4viuL2FIHDEiV z4;y|>_Xh1kr-@#NL8IGg!n&t$<^(puhm|i8Wd#d4{=6W{YT|XfxUL|#^Ymx)#Z?yds;Y)A=3NX z63r}(F;y)HR6wWFVfC<+z2s-;vN`Lu8?3B~i5HoWVo8~D5uKXgmVs3DNL&Ur5>UQ@ z&e%B^QKdM>YOOIf+o1C^43u z!XhW~ZN7Nt+0T3sw%L^Q{aE31w6)Xg+%nBk~bLvGuEQeB0=9{Cu>KJ@)u6x zxb8u@#&zzCq96<;XT7*a=A(llL|8tR^^}_asA{_p8KSfzugLQ>_Qt+CAI$@3i$^LP z!Oa5Q#Lub43U~|#xI*3ZijA(PYi7we2Xd`{7N6oC<{A@W1koV{wJZ3_Kd}h&!XCm# zo|fWkt+4n^Y>&~)6}k=DMr_np{&0UY?8m)!lY8&BK7L258C<~#lfZXGDKXO1j&S0g zlXW);%iZM{TjzMQ4=PU<{YTZxFcN1oY=tx}AmI~&d&r(bYzA9@$O3)2Tl@sqt4BTI zQ}%7NUxy}?p$tay4_*>IeV`sok1on~g_}Fm<_^sr<7qg^B3J|k$u##=I2*FJ>z*Xx z(vp694jsho*6C~v#O-MIp^^Jr8OB8phn0eerbQ;AfM)%*p?#Y`_d81pc@Q&49(`{ zs3kSQz0T9E*dZ$ihNB4|%RsWJ78ee^_uHU+a4{WP-aB)wl1mzol`S1>>2H0SF)1fH zn>w@q{@(gl@|Tca-OnBU{8*U}?a@Woe{XkjGK>v1`nCIrw~dop{e6K{)-N7WhoOA6Gm$q z(Yw`-Z)VEA)VVCEfj$UQTd@Ga04P9zSHJ58n!EPF5W+5iah!R3q_eH!V{<%Xbma@) zeQPHXFz7x$k7eJYMqb2!krAiLPLZQqsE3bW!GA@nwkJ!DAEaz^Rk2lwKJYVo=07o) zX_+6rmU;^RP1IkYR+p<7i&|EuJe`z!=oflVHxRn&7$mB~oDsXP2 zDumsrl31=AgFhn@D!r8SgDZ5Ue^SK`MCFLIe1SjT__lx#TMvTQnegwy&VQi$Z$ z=PPd?n{^r?kTS9O9uQba^N`W(arc#)%FH%Dwdgm03E{%+SG?DQqIR5Y3paIIj2No5 z)MVzKQ3*L4^m5UwdaRATRd)=rO6IZb{8w$H>&+BamGFneehdbYT-fEB#7xki4o_mKWvk z02L*zoZELmV{Wk>gdkj3M|!t-DKN$|B%h+P4K67mXfs@^*GuiOqHX-xDdeg9kuo%P z-Z!+t-jA!c7D+NZXWMQ`ggj>9_M<+x^S|`(fmA64uEJk~x!Z`v&-$9Z?&}cC{ z4(9;wSgW*4(R;kg5p^CLW>?*w6j>!-y25r}TcmAL34&QAwDV9O_GzIhjQ&BN3MZY0 zWK0oHXmBqF0#@Kng-C0%m$CKDd-3)CEf+;&`?+5G{T52)2{FjQ%v)j&fGbb`bEsEf z1{3sDKmTbUHoCEYr##dhvwYUy-ORrx7#kZ~sx_LS;8Yld<-IQ0fy__&QrQ;>muJ7G ze7N5nFoG^3&0D~TT>^S%MW278%`mwe4sn6rp5#M!5r(!!nU-m-ax7A_v3eL@t?nUIzdcmf9=_SE6;cWHCkqUT$-&W`5&_}~Cq|~ZhQ~KSe zGF~%yCw(y&i|U)+oI;Geg&@7$S`JZv_WJ}Mu~rVEfGstaU8JMD)X21vc=9|cB0b5~ zLm*QQhStvs*ZVo881g8W6Qtm-wlNMnPY#PR8?`gCA+1N%LKOmds+Z*M{*4SMmqd`flnzV7zi&3$3=Lh^IG3v7ra1L8hhP$2Hbfhydq=dMMYPx z+%&9ow0baQ%UzP&&vX-19%&0>u$|9j8&)b{4xfK&!e@28c6)xr>gt%h7+kP%Ap7_B zi)n~{=8is z4(awHW(fPHoM&&H$0I8I05+3GA7UNyiwY4G7^bcejuMIff|Hy_$e?J&4SJR!!eCN; zMe-~IAMQRjkR^YfC(XLUM!q#di^W|w3SxiQmzimK;=@QmS0_nwa%Nqy8A91yl^5$E zG7J5zCU2ylS(Jx|A$6)n*>K&)AV`KI0ODjHC?|vF)e9YV3XJ;WxKjfxD)FH$3!xIMTc~4fWA>ahj-V6hU!-&wrU=@rxMgt0V}=- z28K>AUO~fQvT*{`u4Ty}qR-ba`2nc`Q8%4#!y|mB3;b1&dSooHyJ0COD15@C@rmia z1}AfSFNi7lf5Zw2G!XXOq&2${BtqOn8z9BMS{yDtjzJJWH#hp>5sj#ZS^0Ge44XGF zqCBItkABpHaF*UtTJgh27U|shP(pdcARJ;S)1_G8bso(5`JI8JV^+*CY#vOxkJ?gcrM0Lo5V)Ty%c|Qcx&T3?MFA|~f zc51qOjr!>mD|J&kBNa09z#z%RN4{_uQgPkC;GU54aZpoRgrs`iijeh^g=AUJ&F#iC zh|e>8cu`L^u&{=zROl$5zWTa=KX9kE6Ep~`j(TMsrES;<#Q9(=#OtD2pbKHCB~oaL zrA5C*ubqr2u8P)_26Z~Ol~RzxxJ-X3AM|8nJtME-URsObY>fHPnW;nl|h>X8;IwdJ5v8;b83eN8_;1rah&ZSYKUmPfP8jhYvj!z6eD;a2nm22^gmuZ z*G~EW;b@_{4jAcmrKSR_4n0{{ozUzjqE$~5RS%giI)^X3I^*6Ft$s|90s;P}O7O^a zkUdYobdI2v#RiuF{z>PA>OFsKb8Sl(Po!?O_h2{9XsLQ|jjb3rEo2_2*J zKLe2z>hbH=58Y%!Gn&7_5<->My5Hp9Geght>bCS>OJdQb|4(cVi7EaCasJm^y}q7} zl>6Wf$offRl_6jBC)2XGGwAzHPQRpvLbsgC5LXya;EmHV}NA33nj?KtptfE+Un%Edh5DdGmB;<0t9CceQjPZ)zdunb_8w_H zUlde_#{yf2LmPDIcOTMxd(ix6rQ{~^4C}9S9D!z$;(t7^_u0U=r$iC)CtPDl$T|Pt zDM!Y)=wE^jjR($BK{q^1;e4i$$n1*%CiiybGu~q92S)jkgwm@7f6q1{aS8XOWCh<& zkW%si#TyteSR-K=^u%1zTG!1XbpDTdO~xU6$^#1_`k(L2Y=tFoBm8nvWKHGVqbEFo z3}gKLlNk#_1-Y+s1?;85W(@gv)%fq7unoC@MwHPXkQ?w`y`f?z#^S&0EdT|=f@_}^ zLBkHad|oN|e+*e8OOtT2qZd*EV4}f``hE8ER8Nc56RuJ zg)P!2V#=|8Z|LrEQ0pfEsCq~sMHpZSVgdUi2+vz3OifL#jsQ7abg?QmU=F*)*?2g8 zt6MbdfI{OchkPviZ_vVBxttk&268BZt)L>fbrytu8J}7-PjwBT7Avc~A#=aKH*m}% zno|mcDEK!#-e@@3-DG(n)Kydf92wWefEP;TYrTbxgCzy>fxrUpjU@L!3-0ft2aqpEiVXoe zZJ<-md9OZC^nB|6ErI}w^x+SRh2>=U-Y(w_p6LUi3`qHdvC1U?jT0t|f@=Vk0|i~1 zYBDG`!2`6OfnFgA1j|PtckAEhd%C4i)djp!NV1-9uM8FfBF#G|unong`;!nPy}s=9 zGW)^>zHKF0DIM`AwE)VF$oXFL0|7|5x+kGG!4r(n-$Z;||Iq-r%`8NLbX5R8j%CBc)LV4>KS}fMfM9oJr-1-S#BqnKKM%E z(y%r9@HG+}=8IMRaA2DL{`Tc@ens%X0u^kV8-zkm(*+L9FS3w`647Y|KqjCYc-0GO zOyd%(6R+}a2JoR3@M!-=!c^%nVR(R+^}Q3PdK@tG!^N!Vdw##%jLIQg--q;z%?+$Q z1lM6%Lag>&IKyE&q8;mO3b&F#~{S%$hL-TwX;z zxoX3GEP9@`N<7}TTdf0S+Qj}Pn@fVAneOor4Ki)NR|28^d^@V**B8&MNYjE*76&2X zl}YJhz7+I8Mu;sCS-SeG00k!=AT?^~qG5zAfTnjc2s{M%!arjtQU9J2;1W>3g~nji z3<&vvj`kM1YXPKGilvu>h0Bx=NC+T(^1dVip~`KsScf32x!;1rxw|~5lK$}4(KW*W zQM74Rwuj$R^RCc~2FEiy{WRt`zoWz9qHtQr-P1J|O5MV>n zBX+V{!8$JcC?hzp^!=N0{t4|w>j0&|pSw=zAPj4gNZuqmp(I9JgDDHW);kZ${NC1^qL+%y-@W`TPIZ$VX*X?84vYq^rZ|uI$sSL>)Q&R zco)40x}oHeh)Gij)uDXV^SJH2tY!cgaHJyQ0-A8B_3Up9G)d4-CyMUut_0;a2$E&| z*@dE>3J%DXTYMhKlBK^eP(=BGfFhC8gj(~}D?5J!yseeDD(kPG#nTdy6D~|N`I)3b zO(wtv1aQl*$0E~y%N@W#+nMS0Kj3goEwb8AHa;m26UEARy0rYwAM&`~1)lZB4%~43@WKG|`7U7d!m&#nfK4FEt z&(CXXgenHBubC<2eJhhNJpuI!I0GoDKnuLZunFi{a_I=0JT~q@uRptPK{gMKkTV+j z@%|cnO8bSYZ^)23;F{@h+#!IagF4qsFLe*}uqxGp`~{2kGQr<=xInjSNedN=oX82J zK!1P6kB-ek%22N)WK@7yDwy`8Ee zu07Db!;BYjaq5_qO-GTbnIIZVI{L*$A_-|52?>crlsX0(Ma57%3eK`vs5gGD0dl$a z>j#Ej3;?#sYO+;WB}36l+j6ANgHsednj)==ZtmO)Uu@CM*OK$PU}h#b!O`e*dg#Om z*OeN-CJFy}Sp~FlJ*>n{gz?PZ`YreHca)ib+Jsgc)k^Zn?l`F_!4?zO*#7yez2g4_ zFiE8&5d z^G4RHPWezJEFBQ7^4H(R=tip_5DN#C1%(yt{Bmdhd@ujD_m)m*wk%FisFVkV&24aMt&L zhh4sQE-Aa-7t;b$u$;hBhnJXRIdicNa*P?jPGY+kTTFz_fCQ$k;tBK9B!P4NvB0kix8clF+ecE|f=|3fm`9H7gh!3(GO+jBebz zhf}C!t><{ib6?`KgDXeWe6&A5^xX4o{yB%nhj|T173D5*zMz7q|iC zeEHDtgq${lJB8j{4g3p={TbW?VrVYoHe_BtegJG4=1x})-;-gR7J3q3dz~xDUoM=a zH$Cl(fd*ZgCW3bR8z5(}-3MAduo!oQeq|VxnL)6qol*<&b)HzZJOvj zuj`}eU;Xt`7W{?)f9l#ODq@MhdA>>U078vT+KTN(wb669b%OlKfv|v zlSg5JehJcM@A&YL_9y$D@@#|-f_7Inklo43u2wODoCro=15XE>0+WX(wz!(8s2O}e zyr_m!ApX~PPP-v+P=HSdS%DpZWIulV7{gX!PLu%dJ?zw&aEA+NxO4vO5pjcT7V>yA zKBxXv`_hG(lhP)1i~7UEGvqZ5j`+h=DNr1%`~mh!ns{yms?70VU2%y8D}NJeCsI<0 zaETYn%o>Wclq5~L4mW%~$ptz<5mj-W9mDqK)fDE)@bK~{+!Ka)gYd#hPMBl zmTCU5OcGdfM^My^ilAWFm)1bH%yiCCd7+l%&WACrHhemN)tHlCleQq95v_`+YjWz6 z%OD5bCml&QQ#xPM%Ux=H#-IG$&TO)_mI>0ONf}&o?e4iC}&7a{k z$TcOsVUS^uq-!iY#3k!cU&SGMF9@h>16cK7JwjXG{zm_SqJ7Exz!J`VU2CY_ye~E# zzgInM;%Cgff|F;jp%G*8QG#Av3amhTT+3?(yfZ8uYagUd-KeZMLO1`j^U+jMh?w$Y zw1BEdA?{NxlV!Ro)xHjOGfnhq9yp8CPm;O?8F90^jRn-N*!+_SWFk{>F>loCTs>qv zIPuHW^lT>_@r)Q-3X1b!Wn#sdT+sP0dFCFA(>bnG$9ULNKr}*DXy!M2qAamS7o14GZoo4{YVI_Ma zLoT>Kf~nIRQ75&J-8k-UhwIZhjshW7|J=rmKOhL;oG6r2td}$voWZM9ydQtH6dYH! z=U$lTX_rdEQ8u_DyM}0Pj?WMywJd5VIY_p}q%IE4Lu!zpLq&AhC^R%Q0J>t%w#XSH z&@eHknHkjdXyTP0&LA{3Ypb0npwDB`xg~M-+OV}D+RtjW_^7#mq~#X3i1d* zC~fn=A|_@z=X)|nTilLQP=4SWuB^ec6R+QT7UGvY=d|Ax8D&1^3ixx9;b0)4v>63l zA2*09j~?xI$KU!bd-W~QS%WNmuZ37!Mh=nYl)SHT+JLX1JRlYknjx^D9Bv$j<8Q`? zhZx!f0Ei1^nq6`Pk@75Zd+!$|5N2N`B+Q@@2I=nyfDN@0N3SEk7_x!bSZZ)pQ=X73PihD<&-6g$Y^Z0ROBe|yooaaq7WCEKf z8+|X@Oh(ZZ4T16oPzLRNa5|0T++hleAw$BUf2gt42gUbH^h&o*6H3Cp@BG};_RX_{z*MxUADF%{EMNVeEClSRpOnH~jYQJkm1V67IG|fK!8D^+hY|M$!U#6n1U-d5=j5U zas9va|41Oqis26-S{DC5e>prLVKj2zA^$^q`d_2$Ls@BPZ5G{$l#zLgijF}*?E~KA zPsIPcPVtoPsckac9O`aR0o?#Tg!E@9=PJDBL$nJG&M-_yCNAJpz@vJ4f)>5q*#pJ%QZ~!A2v%S8(z=t|Xj75-KGG zh?mWH^%ii3H1Tvv4uOUtA4&Gqk6ArEQXV3(V18XJTwK{HV1pjQyFs4HxZ=TAq&Xf9 zDQPeRzjYrPb?Oq5vGEw`=nSM;nO~ZAz!s}gDv*MpZ+j|)CR0jd6=IP}FVK>N7=8?@ zk_nqoLM}0?(Ml;O2)axo=Isi;9F7TKt7f|cbnrKNU}hq?RfOr4{$zg?va3cIG5nAjsK3TD+L=v4kyFp*>3zdf zBrr6WsIP;a0Tyq-?$S!>xygz|&_+Ih(;&FKXsLBv49x?_3liYqrLba}?#2hSK-Ul( zc#Qy;Cc{AxZD0TncRD4kUu7aXm)OEkRT5ZH!sdp;z97RZ8!box5YNew{pHuIHQ8r99dM4G7_pXv{=f zPNIiOY4|)#o(`Dmstvs@XiJB5_hrCE=T~z zC2Su|boAznVWyNu%X{uAng6HZVg2|OY~c3FEW^wvE`VE72n!ci&yhhf z$qj=|B!nAyHIAjSW`{GC}4VeNL8S;%LuuW0qU!s!m zrAmjVv$@4 zI5T2Fa8x`Sg5=+f#5rKPiNBw1~M^gMCXPcB8QG6HNP zMBc;C6ao&bSZ5Z!knc?8YrK*EzWL6eOq!?l4N^yuK?`ynRpW`-(OVy!+lnshKGPPs zdksY_)h1Xyo9{NHWWXSNS7-_+rEajlzdtxCK~JO+TTYt2nH`}9P_08$5RzvueU%CWn{J=3%qkfQ zP!?TDfEliQe0p*E4tQNle}@8y#3-F8Dc@}f__B-Mi{-wI{sXxf!EF6AJCY}=-YgL8^i6H`J`8VqSgXvPzsTE z%Po{%{-h=*juUcuFxy+i_bw0eC%{1mN!;N^!i7pp{TLOW4M}IDJKoI32B(X}IET@^ zUU9V>y?clYJYeTVrVG8OOgZUA;g!KeH%Vf9`iRNMLQ{duq(N;R0~H5-uZ){hAeio9 zS>d@gt#M;aL=h++3c&>8hm!nJy5WilEVvov3u=dk*h@n9q3?%o3kN?ii16e-@+w{A z#r0fB^Pqb5w$S%vO~6hsU#qRVn;->iMIUZGAhat%L+Ujx0PIlc-uRZ5XuLrkKzAR9 zdx7P(PaMn=(j&Ys7=F;Eb?xl!nk|Qv_}EwP`cn3^lyyR1Lu}R z#PO$OB1_-7xn*Aq?C(0S9p-^vaM&)o$JyWH=SM~28~U;F$%!!U<*3wXN;GNkD*nlkLAZP9oVtXl`cMNFoCZ1F1iY1WXe!l-MP7QIxq>+kP%@B8(7Z|H! zOC0lwED*6^(YZ7KuuXp4EU*Dg&_Y?$0i@k&z_LG9Y%( zXhBL4L_|O(hLV(&Zd9aES`mYg5|II z_X5pR;!6yAWMTpeGMOy#;1YDq=%*=LR1;f5w#yNOS}amVOG-@C7d?&yh~ZJOU~NUa5Z?O&NHnrDKt7 zJwb$ZxcuV%Yw0JyqXznJlZc72ey{a{Y|BS?eMLfIVqydWT;=yG{bla1SenQuRqywq z-7Q=&5C+s%9?+j%aQ&aTpT2D1XE;7_NrSyNNs9fQn%EbFQh8u6u&TW_ z_>4{4I54N?U10+IJ+*wb>ixZ)cjR5^hN+_0uw^!dw@i<}A@oV&+r~Q$d`K)<0?Dbz zRQ5n*FLGWs%ZYFpu{b{qj59dyhu0e*$uzwpY`72-a9m(?YAU(YP|ZnKdiA^aC%uH> zVG}DgzjjExDj?U-zi*&A^z-^C50H&%{HK9_@r@00`_Y*RQc{n#U}?&7p229=!?Ubn zg~D)PwJhbdYy~v^&_%9rgK&k>)s9VpQ;@2wKW~d zu={?6E>v5r=Y|+}_}#0jKM=i>SN61bVXRosu;^t_X_b!9lX)(2RyLqZP-r$NPWfRc z`e2hwG{JDA`<%<_y8Y;K+K`M`ezU}GAt9E#QU5G%w6bi}G={swT7S~h^M34)EA}uY z4m(W8jM*;P^fzXHLYj7iS*i#c>< zjJ0lEpeppb*)#N6mMRIL5jh*=Ebpg@;+Rif`q3?Qzs}A*IZ=Xu zg%?$e)m%gje`+rCu*@B@{rK#sx-oj8yb7MnZ5NJYN1773td8c`zg=qH!7$LA(f3jP z?4iWGU^5uaLQ(vrFF5)$v`VU^8ymP`BQM!MMTUh<0!!&Q5+8L~(1SMlKZ*|jcz=k? zqAV$nYMjRp9Xl+wX6`bxaz;hizD*rIK$au%qlZg z$p~`YQf>Yk>x`Es?}m}I*{NV@B-I$9ir<+BfJSY_&2dBx-?I;C+vdNpaW-tNB8ArV z3JWi>0?QkJ2R!yFQrQ$!y{LP6`L$6*?RusCugyc_6@QEKaMlbeU+1pS^qcGwK2BxE z%nFmIzM^~<6hrecs~(Kr^D57m{&*q|JTbq9BAR?B)LEeMC5<}-FL3v#mX> z_>JI^Pse&!T_q|bkgx7>CGtNZOx)H_uV5lQFEiT7BSBIOK0TIm3h5|Xe;7Q^jF17~ zYJ2i*mfAl&C{9SUIYGP+s%jzc3PDx{&trHTO!9Pp6ngVQCFbZ&vyQ==-p1~S`}Gof z3yT3#ZOA3UWJrh|c@bON+uJ)kiISd1LyT-)|LMIcb5Sv~$I+o7AuY3=sZh%ciLNwz zQp!2DO(t4u^QH4_G{YRSk8Byeh_cB1Z=f69=Q_ZtG()_Bum`djV0Mata$Qj&^-s%s ziyEQ5BTDH;7Wgs?LrHn^!KwF!0>Kq8dO*~IujQTl4{!2se#I9?&nK&A^yk|TR(hX= ze#J@3E9>zFY5Kp~l3W=uFS*8O?>u5{!!>WB7yN2e>?oHsltjYRPsIE}+;x&RJfY$6 zH0Q;L&)a32HoiSvxi5!xbinCu%=|B?D3~WDq2v32KK6LJm9;+Gs*UUcP&uR*I`bC# zfGBa*ZPUrzE6beG+b%eQ$#37=NGDbAgt=Cr{>FSvX^9s79@D}eW^TkAW~}3xw`pLvR6Ct*C6uvup8-H2hC^? z5)v<{lLiWWR4x~Ga~5Jwq|p9^nC)IdbLuv*sVAsZ+te&i2MHPZSRU+e)$k=*L1zo! zuYB82YIy4=Z_ap4OHRu{K*7UaodOM}UeRXq5my3h+oxqc)?UxLZ)4%h^^0+TQoBIc75rLP3c6Dl+%UJTV&Dj%QpY{q_Vv{D)|^6pKC)0lQxCaH8jos0CD4-Huv=b>riwhaO~IIbI3*E+>Ymm5HF$NfNXdJu(G_ z$b{BQ)5My8?28@}@sz%Zc)kLM*-*&d%!l@Gc-DBQ%O$9MTa9{p->@=)cN+&x9iZ$;bll3KD7nr67< z{(5_Ff7_N1^;$c5|7XO5)k(cT!E4tTeoNxxAWnef>U?Hl>4PmNzCg<2QuX*;^fwt_ zntJDlYtvd2RpdWYG@tymX^wl>qwF5@a^$UTcd&7`MVtfSbAqi5;~~b$FZG{P!%a37 zqErzVzPYTPVx(7^+1<)5(g|O3il;M2W%~IlA-QdwH|A{&S?b6n%Z|UU9kpidsbEPr z+)<8FpgYuIYyv7_tXp@bknA&!!c26D=S3e`=t7@1#*m9fyIW(mQPwBJDk#2v&`lKr z(1rEa8-di1r*xJE@BLi-zBV&tqufegdp6emBf%TnXt-CeIhITEBw_7pJDsTsRfL2$ zf#zmf9}ZPDJ|{h&6+z$$g|F%{A96(yg>GgKUi8ij}=?6OcyvpDk#u2 zj*XC!%`{C^jT6q4^76$Q?Jq>Lq0#6^ ze%t%Y+e~HLZ9<%S+%v7R_@*J}j#bft>iTj!pYovx#_;soETqfb)-=cZ3$sHQe^H&$ zDx%xh6ji$M;@QO;>6<-S#=F~dQBZ~@&9pGy3u0U)bh&7)F-E5KBSptBmw{}U9@=fw z6b*pFo!Vf!I`23R8sqe2} zyf-UqAW8IDEYoT5^NA^l=)E)S5rrH$-f8^~-&Xe0`OO+HN~2Z2Z>w?c zp~8<*YrV8>zF85I3I$)rYl{Q& zB62~euI!gkPHFJ{c3amah)^gpOO&|RDP95g$kjF3XJ1qDvd69i6*RBkvduls2bkEP zsdJ=??Pj|ieu%c$FRX9!HBj1gM4%CdqhQtMsJU#ETX%Ft+{G<%-sjahQ zI*|?MD#h^-D2-*17F^>!@*A9FHOO(rSi#)BA(Aw=@;R^&(<(wbqaQN;Prl=O>i_{4 z@eK6_GC4-%5q_0mkvUz^IUYbq_~juL@g;{^KQ2($Up7gfjg z?dQ~%o0>@j=S~XlTG&R zd<^4=CrkUAh(qm+A3uXq@(YK?8Z@e3PSribVP^5RlAcT6LX(N-xUl|V{jH#3*+B&X zlME`thdv0aDaIQjnnr@l-1|No26`p!ep4d}KfBz6i{`e9el*qPhB1tS4y}L=(`)*m z<56vCDxCE4JKAw_BpszYJ9UC?&iOpu@7*>f6@s~o#O@0;)&x-%KX%EN4}!te)Jjp8 zgLu8$4*jA-N$S?Cv2^MBx|Cxaup&M|^Q5n9llrq=`-cp<(-Y0{d$r821BKZ#=jjx9 zoXHzR4f0jr-aLLh(kg(@hL$7z0rgWA2*h=m1eoP-pF(M;p~8w;QxXJ2__&6au0HBJ zJ1H{}$`XGjy>D!j`6(HJ`(mNxxSq=gb$-@Euh%aSMMSc`x}g@(Qj%f+BHufLuJ&9! zDs1!N?&dyUh+NFZ)C2S3>(JpAy8&X>w2wnzGW1vgP=D39^mbvM9J4~BEv-w|W0PUx z-9+GE8#2YCDluH!oG`OOXBUog82UWC7nG6x%YK8{%iAhP?sAL;F7}apntaPp&YJ5u z{EQ6$a!iO(o}FecGNzAWezGY|#@B=0K3*-n#&&sBlk*}giMNB|jjR}c-xRU?-mdW| zXu!!?$L~0Qx>u~XkyIVtxI@4%;@wJPjBX0$hm+5#z>gzsvfIC&Xd!zTf{@dZ;M(^5 z9+3_LgC&Hu#!_=gguJQUVh=C6AcODdRkPZ0RRMLl-3Jp(YMhdvj#$gKypoGTg9!y}u|m@%S-^K|`8$UpMA%OrT4Hb!ULmcu7OsfvYiD@!lD1wY;Rm19F3%Q|mqmWJ zrUF_83EC%&W^P#U2454bRU$*S@$76g$xs%>toWg<2TSr+3WAPt+eAY)1%;Amy8v!D z_^{dny2b@e}$I?1@U^K)<17T({Tzt0=pF|e>MYKhyfwMo= zoK`tJu@J9^Va#k3#{b30C9Z)~DRV3ADjt0LDHqh?k9H|gNX4E*_MKmhm7+W-VvIcs z{dx!P;hAWng$=?VPn5>w+uKL!*5y2szk9I-6pCZ%oa&>on)Q|#{s8WdugFHBAvey6 zVJ_&a8w64t$sTlCjLDb+jSZ9k5RLB!S2!ylycTtGM97X-cK3hxk({1fV-RTESm^NF z&5HUO=8Rcv-dJE@=>s=J@rwiA9K;{Mm;Z4UP!;Sr*`m+BoY3k<5 zgvqt#!GKztL?d-~RjHh;Ld{6@xwB{839=%Sj>9ik1!Z1!$raGeb;A*Bm7?kMHVDhvnh zR?DwGSoxR^(<-tT!5?b+u|WUsX4Z3X#;)A@=G(5EPaN9(mr|rbBJ64cVHjcfW-REG zt-91sG+ZdNACcnvWM|bA82OrqQ?)PAnlM)0*$^XTIk?mGM)Nk8>;eaYhy79x$jRv` zZ-QcqGT`$udIZ|N$H}p8f6UO&Ab_aycT2O?#JGuhl+USjpY8HhqTA@S^SB!(+T<5_ zXgQT&0K~}mqKn%2&_JW)USGZ`bZ7!W9sB8h$21*>&>?nXux`k+#diuu%@nPUESmeb`p2xDFo81MeJ8sZeU zY}TlvF-?~cvC>drlpc8alkOV}^;PC-v(@2B9}Xq)UW&=5z5^|i7`X(ORaAXQ(vPT+ z%1_&+U1^6H!c96K6Iz+yg~w4OOfoLp7{m+s^2{XW5<}D??(Rv;kw-sSa%qNzW-u6 znuL|AzhgkvqOJ?MQNYoi+*lWGvpQ6PKY3vH6#kC-EdOv^jGK8N-es%u(&*5t{Z9ns zr7`X$)?>R_KtD4@^R-!rUWZ*X0X8j zR5be&Wuyemj~~0yB!DvJ0GT<>sI(WLI4EP#rs5D5h&!uBb-{Ie&DxzRRQNnEz>XE5 zC{@s!ezJHFFprXx(bo(O4Rv=r0eA=bvs%k1O3!gG_-LbvMv)HRrdkrag>d|}v8WjF zp+#c!@w*LT1m(7^JU|FoUtI+qiYxI^LP8>&ic$Dt9K9NOvVS-UF>(7pe-S)oEH(4@ z*4j~fy>>hTHR5CxRh9MV9dB>%byca+*LXv?Nyo7u699*!G_W&Omza<$8XDvhG1%u~ zFtr3vl>fF=Y@0XsvkppFdG#ud3cdL(iQM5N_8n=+=8Cz_8{|j#6Zn^*f diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index ee16fcba3ba..65ae5096e76 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -355,7 +355,31 @@ Imagine the function executes successfully, but the client never receives the re This sequence diagram shows an example flow of what happens in the payment scenario: -![Idempotent sequence](../media/idempotent_sequence.png) +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt initial request: + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + activate Persistence Layer + Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. + Lambda-->>Lambda: Run Lambda handler (event) + Lambda->>Persistence Layer: Update record with Lambda handler results + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record with result + Lambda--xClient: Response not received by client + else retried request: + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + Persistence Layer-->>Lambda: Already exists in persistence layer. Return result + Lambda-->>Client: Response sent to client + end +``` +Idempotent sequence +
The client was successful in receiving the result after the retry. Since the Lambda handler was only executed once, our customer hasn't been charged twice. @@ -367,7 +391,23 @@ The client was successful in receiving the result after the retry. Since the Lam If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. -![Idempotent sequence exception](../media/idempotent_sequence_exception.png) +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + activate Persistence Layer + Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. + Lambda--xLambda: Run Lambda handler (event).
Raises exception + Lambda->>Persistence Layer: Delete record (id=event.search(payload)) + deactivate Persistence Layer + Lambda-->>Client: Return error response +``` +Idempotent sequence exception +
If you are using `idempotent_function`, any unhandled exceptions that are raised _inside_ the decorated function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. diff --git a/tests/functional/event_handler/test_api_gateway.py b/tests/functional/event_handler/test_api_gateway.py index f1fb6a1f942..4b1d7c1ee32 100644 --- a/tests/functional/event_handler/test_api_gateway.py +++ b/tests/functional/event_handler/test_api_gateway.py @@ -283,7 +283,7 @@ def test_base64_encode(): @app.get("/my/path", compress=True) def read_image() -> Response: - return Response(200, "image/png", read_media("idempotent_sequence_exception.png")) + return Response(200, "image/png", read_media("tracer_utility_showcase.png")) # WHEN calling the event handler result = app(mock_event, None) From 631370d47f1fce83d9a6cb119d337ce8e294f57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 16:03:33 +0200 Subject: [PATCH 11/26] chore(idempotency): remove param `expires_in_progress` --- .../utilities/idempotency/base.py | 8 +- .../utilities/idempotency/config.py | 4 - .../utilities/idempotency/persistence/base.py | 23 ++-- .../idempotency/persistence/dynamodb.py | 50 +++---- docs/utilities/idempotency.md | 110 +++++++-------- tests/functional/idempotency/conftest.py | 79 +++++------ .../idempotency/test_idempotency.py | 126 ++++++------------ tests/functional/idempotency/utils.py | 39 +++--- 8 files changed, 172 insertions(+), 267 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 821e2d46f6b..ed9fb8b6dfd 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -190,12 +190,8 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] raise IdempotencyInconsistentStateError("save_inprogress and get_record return inconsistent results.") if data_record.status == STATUS_CONSTANTS["INPROGRESS"]: - # This code path will only be triggered if the expires_in_progress option is enabled, and the item - # became expored between the save_inprogress call an dhere - if ( - self.config.expires_in_progress - and data_record.in_progress_expiry_timestamp is not None - and data_record.in_progress_expiry_timestamp < int(datetime.datetime.now().timestamp()) + if data_record.in_progress_expiry_timestamp is not None and data_record.in_progress_expiry_timestamp < int( + datetime.datetime.now().timestamp() ): raise IdempotencyInconsistentStateError( "item should have been expired in-progress because it already time-outed." diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py index d77a6dbfe0e..06468cc74a7 100644 --- a/aws_lambda_powertools/utilities/idempotency/config.py +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -9,7 +9,6 @@ def __init__( jmespath_options: Optional[Dict] = None, raise_on_no_idempotency_key: bool = False, expires_after_seconds: int = 60 * 60, # 1 hour default - expires_in_progress: bool = False, use_local_cache: bool = False, local_cache_max_items: int = 256, hash_function: str = "md5", @@ -27,8 +26,6 @@ def __init__( Raise exception if no idempotency key was found in the request, by default False expires_after_seconds: int The number of seconds to wait before a record is expired - expires_in_progress: bool, optional - Whether to expire units of work that timed-out during invocation, by default False use_local_cache: bool, optional Whether to locally cache idempotency results, by default False local_cache_max_items: int, optional @@ -41,7 +38,6 @@ def __init__( self.jmespath_options = jmespath_options self.raise_on_no_idempotency_key = raise_on_no_idempotency_key self.expires_after_seconds = expires_after_seconds - self.expires_in_progress = expires_in_progress self.use_local_cache = use_local_cache self.local_cache_max_items = local_cache_max_items self.hash_function = hash_function diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index a2548d18e8b..920f93bdae7 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -125,7 +125,6 @@ def __init__(self): self.validation_key_jmespath = None self.raise_on_no_idempotency_key = False self.expires_after_seconds: int = 60 * 60 # 1 hour default - self.expires_in_progress: bool = False self.use_local_cache = False self.hash_function = None @@ -158,7 +157,6 @@ def configure(self, config: IdempotencyConfig, function_name: Optional[str] = No self.payload_validation_enabled = True self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key self.expires_after_seconds = config.expires_after_seconds - self.expires_in_progress = config.expires_in_progress self.use_local_cache = config.use_local_cache if self.use_local_cache: self._cache = LRUDict(max_items=config.local_cache_max_items) @@ -353,19 +351,18 @@ def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Option payload_hash=self._get_hashed_payload(data=data), ) - if self.expires_in_progress: - if remaining_time_in_millis: - now = datetime.datetime.now() - period = datetime.timedelta(milliseconds=remaining_time_in_millis) + if remaining_time_in_millis: + now = datetime.datetime.now() + period = datetime.timedelta(milliseconds=remaining_time_in_millis) - # It's very important to use math.ceil here. Otherwise, we might return an integer that will be smaller - # than the current time in milliseconds, due to rounding. This will create a scenario where the record - # looks already expired in the store, but the invocation is still running. - timestamp = ceil((now + period).timestamp()) + # It's very important to use math.ceil here. Otherwise, we might return an integer that will be smaller + # than the current time in milliseconds, due to rounding. This will create a scenario where the record + # looks already expired in the store, but the invocation is still running. + timestamp = ceil((now + period).timestamp()) - data_record.in_progress_expiry_timestamp = timestamp - else: - warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left") + data_record.in_progress_expiry_timestamp = timestamp + else: + warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left") logger.debug(f"Saving in progress record for idempotency key: {data_record.idempotency_key}") diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 738c32c7126..bd253b4a3e5 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -168,35 +168,19 @@ def _put_record(self, data_record: DataRecord) -> None: try: logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}") - condition_expression = "attribute_not_exists(#id) OR #now < :now" - expression_attribute_names = { - "#id": self.key_attr, - "#now": self.expiry_attr, - "#status": self.status_attr, - } - expression_attribute_values: Dict[str, Any] = {":now": int(now.timestamp())} - - # When we want to expire_in_progress invocations, we check if the in_progress timestamp exists - # and we are past that timestamp. We also make sure the status is INPROGRESS because we don't - # want to repeat COMPLETE invocations. - # - # We put this in an if block because customers might want to disable the feature, - # reverting to the old behavior that relies just on the idempotency key. - if self.expires_in_progress: - condition_expression += ( - " OR (" - "attribute_exists(#in_progress_expiry) AND " - "#in_progress_expiry < :now AND #status = :inprogress" - ")" - ) - expression_attribute_names["#in_progress_expiry"] = self.in_progress_expiry_attr - expression_attribute_values[":inprogress"] = STATUS_CONSTANTS["INPROGRESS"] - self.table.put_item( Item=item, - ConditionExpression=condition_expression, - ExpressionAttributeNames=expression_attribute_names, - ExpressionAttributeValues=expression_attribute_values, + ConditionExpression=( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), + ExpressionAttributeNames={ + "#id": self.key_attr, + "#now": self.expiry_attr, + "#in_progress_expiry": self.in_progress_expiry_attr, + "#status": self.status_attr, + }, + ExpressionAttributeValues={":now": int(now.timestamp()), ":inprogress": STATUS_CONSTANTS["INPROGRESS"]}, ) except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") @@ -204,16 +188,21 @@ def _put_record(self, data_record: DataRecord) -> None: def _update_record(self, data_record: DataRecord): logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") - update_expression = "SET #response_data = :response_data, #expiry = :expiry, #status = :status" + update_expression = ( + "SET #response_data = :response_data, #expiry = :expiry, " + "#status = :status, #in_progress_expiry = :in_progress_expiry" + ) expression_attr_values = { ":expiry": data_record.expiry_timestamp, ":response_data": data_record.response_data, ":status": data_record.status, + ":in_progress_expiry": data_record.in_progress_expiry_timestamp, } expression_attr_names = { "#response_data": self.data_attr, "#expiry": self.expiry_attr, "#status": self.status_attr, + "#in_progress_expiry": self.in_progress_expiry_attr, } if self.payload_validation_enabled: @@ -221,11 +210,6 @@ def _update_record(self, data_record: DataRecord): expression_attr_values[":validation_key"] = data_record.payload_hash expression_attr_names["#validation_key"] = self.validation_key_attr - if self.expires_in_progress: - update_expression += ", #in_progress_expiry = :in_progress_expiry" - expression_attr_values[":in_progress_expiry"] = data_record.in_progress_expiry_timestamp - expression_attr_names["#in_progress_expiry"] = self.in_progress_expiry_attr - kwargs = { "Key": self._get_key(data_record.idempotency_key), "UpdateExpression": update_expression, diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 65ae5096e76..78089051ff0 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -370,12 +370,12 @@ sequenceDiagram Lambda->>Persistence Layer: Update record with Lambda handler results deactivate Persistence Layer Persistence Layer-->>Persistence Layer: Update record with result - Lambda--xClient: Response not received by client + Lambda-->>Client: Response sent to client else retried request: Client->>Lambda: Invoke (event) Lambda->>Persistence Layer: Get or set (id=event.search(payload)) Persistence Layer-->>Lambda: Already exists in persistence layer. Return result - Lambda-->>Client: Response sent to client + Lambda-->>Client: Response sent to client end ``` Idempotent sequence @@ -386,6 +386,59 @@ The client was successful in receiving the result after the retry. Since the Lam ???+ note Bear in mind that the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, consider splitting it into separate functions. +#### Lambda timeouts + +In cases where the [Lambda invocation +expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), +Powertools doesn't have the chance to set the idempotency record to `EXPIRED`. +This means that the record would normally have been locked until `expire_seconds` have +passsed. + +However, when Powertools has access to the Lambda invocation context, we are able to calculate the remaining +available time for the invocation, and save it on the idempotency record. This way, if a second invocation happens +after this timestamp, and the record is still marked `INPROGRESS`, we execute the inovcation again as if it was +already expired. This means that if an invocation expired during execution, it will be quickly executed again on the +next retry. + +???+ info "Info: Calculating the remaining available time" + For now this only works with the `idempotent` decorator. At the moment we + don't have access to the Lambda context when using the + `idempotent_function` so enabling this option is a no-op in that scenario. + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt initial request: + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + activate Persistence Layer + Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. + Lambda--xLambda: Run Lambda handler (event).
Time out + Lambda-->>Client: Return error response + deactivate Persistence Layer + else retried before the Lambda timeout: + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + Persistence Layer-->>Lambda: Already exists in persistence layer. Still in-progress + Lambda--xClient: Return Invocation Already In Progress error + else retried after the Lambda timeout: + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + activate Persistence Layer + Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. + Lambda-->>Lambda: Run Lambda handler (event) + Lambda->>Persistence Layer: Update record with Lambda handler results + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record with result + Lambda-->>Client: Response sent to client + end +``` +Idempotent sequence for Lambda timeouts +
+ ### Handling exceptions If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. @@ -483,7 +536,6 @@ Parameter | Default | Description **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired -**expires_in_progress** | `False`| Enables expiry of invocations that time out during execution **use_local_cache** | `False` | Whether to locally cache idempotency results **local_cache_max_items** | 256 | Max number of items to store in local cache **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library. @@ -897,58 +949,6 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer): For example, the `_put_record` method needs to raise an exception if a non-expired record already exists in the data store with a matching key. -### Expiring in-progress invocations - -The default behavior when a Lambda invocation times out is for the event to stay locked until `expire_seconds` have passed. Powertools -has no way of knowing if it's safe to retry the operation in this scenario, so we assume the safest approach: to not -retry the operation. - -However, certain types of invocation have less strict requirements, and can benefit from faster expiry of invocations. Ideally, we -can make an in-progress invocation expire as soon as the [Lambda invocation expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/). - -When using this option, powertools will calculate the remaining available time for the invocation, and save it on the idempotency record. -This way, if a second invocation happens after this timestamp, and the record is stil marked `INPROGRESS`, we execute the invocation again -as if it was already expired. This means that if an invocation expired during execution, it will be quickly executed again on the next try. - -This setting introduces no change on the regular behavior where if an invocation succeeds, the results are cached for `expire_seconds` seconds. - -???+ warning "Warning" - Consider whenever you really want this behavior. Powertools can't make any garantee on which state your application was - when it time outed. Ensure that your business logic can be retried at any stage. - -???+ info "Info: Calculating the remaining available time" - For now this only works with the `idempotent` decorator. At the moment we don't have access to the Lambda context when using - the `idempotent_function` so enabling this option is a no-op in that scenario. - -To activate this behaviour, enable the `expires_in_progress` option on the configuration: - -=== "app.py" - - ```python hl_lines="8" - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - config = IdempotencyConfig( - expires_in_progress=True, - ) - - @idempotent(persistence_store=persistence_layer, config=config) - def handler(event, context): - payment = create_subscription_payment( - user=event['user'], - product=event['product_id'] - ) - ... - return { - "payment_id": payment.id, - "message": "success", - "statusCode": 200, - } - ``` - ## Compatibility with other utilities ### Validation utility diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index c1fd5a8a2ef..bb58597af0d 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -75,24 +75,28 @@ def default_jmespath(): @pytest.fixture -def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key, idempotency_config): +def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key): params = { - "ExpressionAttributeNames": {"#expiry": "expiration", "#response_data": "data", "#status": "status"}, + "ExpressionAttributeNames": { + "#expiry": "expiration", + "#response_data": "data", + "#status": "status", + "#in_progress_expiry": "in_progress_expiration", + }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", + ":in_progress_expiry": stub.ANY, }, "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", - "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", + "UpdateExpression": ( + "SET #response_data = :response_data, " + "#expiry = :expiry, #status = :status, #in_progress_expiry = :in_progress_expiry" + ), } - if idempotency_config.expires_in_progress: - params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" - params["ExpressionAttributeValues"][":in_progress_expiry"] = stub.ANY - params["UpdateExpression"] += ", #in_progress_expiry = :in_progress_expiry" - return params @@ -106,64 +110,61 @@ def expected_params_update_item_with_validation( "#response_data": "data", "#status": "status", "#validation_key": "validation", + "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", ":validation_key": hashed_validation_key, + ":in_progress_expiry": stub.ANY, }, "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", "UpdateExpression": ( "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status, " + "#in_progress_expiry = :in_progress_expiry, " "#validation_key = :validation_key" ), } - if idempotency_config.expires_in_progress: - params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" - params["ExpressionAttributeValues"][":in_progress_expiry"] = stub.ANY - params["UpdateExpression"] += ", #in_progress_expiry = :in_progress_expiry" - return params @pytest.fixture -def expected_params_put_item(hashed_idempotency_key, idempotency_config): - params = { - "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", +def expected_params_put_item(hashed_idempotency_key): + return { + "ConditionExpression": ( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), "ExpressionAttributeNames": { "#id": "id", "#now": "expiration", "#status": "status", + "#in_progress_expiry": "in_progress_expiration", }, - "ExpressionAttributeValues": {":now": stub.ANY}, + "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } - if idempotency_config.expires_in_progress: - params[ - "ConditionExpression" - ] += " OR (attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" - params["ExpressionAttributeValues"][":inprogress"] = "INPROGRESS" - - return params - @pytest.fixture -def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key, idempotency_config): - params = { - "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", +def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key): + return { + "ConditionExpression": ( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), "ExpressionAttributeNames": { "#id": "id", "#now": "expiration", "#status": "status", + "#in_progress_expiry": "in_progress_expiration", }, - "ExpressionAttributeValues": {":now": stub.ANY}, + "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": { "expiration": stub.ANY, "id": hashed_idempotency_key, @@ -173,15 +174,6 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali "TableName": "TEST_TABLE", } - if idempotency_config.expires_in_progress: - params[ - "ConditionExpression" - ] += " OR (attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" - params["ExpressionAttributeValues"][":inprogress"] = "INPROGRESS" - - return params - @pytest.fixture def hashed_idempotency_key(lambda_apigw_event, default_jmespath, lambda_context): @@ -218,7 +210,6 @@ def idempotency_config(config, request, default_jmespath): return IdempotencyConfig( event_key_jmespath=request.param.get("event_key_jmespath") or default_jmespath, use_local_cache=request.param["use_local_cache"], - expires_in_progress=request.param.get("expires_in_progress") or False, payload_validation_jmespath=request.param.get("payload_validation_jmespath") or "", ) @@ -228,14 +219,6 @@ def config_without_jmespath(config, request): return IdempotencyConfig(use_local_cache=request.param["use_local_cache"]) -@pytest.fixture -def config_with_expires_in_progress(config, request, default_jmespath): - return IdempotencyConfig( - event_key_jmespath=default_jmespath, - expires_in_progress=True, - ) - - @pytest.fixture def config_with_jmespath_options(config, request): class CustomFunctions(functions.Functions): diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 6f0164bd145..9b1dc2b048e 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -46,10 +46,8 @@ def get_dataclasses_lib(): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -100,10 +98,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -158,8 +154,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -230,10 +225,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -270,8 +263,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -323,8 +315,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": True, "event_key_jmespath": "body", "expires_in_progress": True}, - {"use_local_cache": True, "event_key_jmespath": "body", "expires_in_progress": False}, + {"use_local_cache": True, "event_key_jmespath": "body"}, ], indirect=True, ) @@ -345,14 +336,12 @@ def test_idempotent_lambda_first_execution_event_mutation( stubber.add_response( "put_item", ddb_response, - build_idempotency_put_item_stub(data=event["body"], config=idempotency_config), + build_idempotency_put_item_stub(data=event["body"]), ) stubber.add_response( "update_item", ddb_response, - build_idempotency_update_item_stub( - data=event["body"], config=idempotency_config, handler_response=lambda_response - ), + build_idempotency_update_item_stub(data=event["body"], handler_response=lambda_response), ) stubber.activate() @@ -370,10 +359,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -412,10 +399,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -458,10 +443,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, - {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, - {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, - {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, + {"use_local_cache": False, "payload_validation_jmespath": "requestContext"}, + {"use_local_cache": True, "payload_validation_jmespath": "requestContext"}, ], indirect=True, ) @@ -511,10 +494,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -576,10 +557,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -617,10 +596,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -658,10 +635,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -696,10 +671,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, - {"use_local_cache": False, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, - {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": True}, - {"use_local_cache": True, "payload_validation_jmespath": "requestContext", "expires_in_progress": False}, + {"use_local_cache": False, "payload_validation_jmespath": "requestContext"}, + {"use_local_cache": True, "payload_validation_jmespath": "requestContext"}, ], indirect=True, ) @@ -735,10 +708,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "config_without_jmespath", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -794,8 +765,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -853,8 +824,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -905,8 +876,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": True}, + {"use_local_cache": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -975,8 +946,7 @@ def test_data_record_json_to_dict_mapping_when_response_data_none(): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -998,8 +968,7 @@ def test_in_progress_never_saved_to_cache( @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": False}, ], indirect=True, ) @@ -1025,8 +994,7 @@ def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_ @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -1082,8 +1050,7 @@ def test_is_missing_idempotency_key(): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "event_key_jmespath": "body", "expires_in_progress": True}, - {"use_local_cache": False, "event_key_jmespath": "body", "expires_in_progress": False}, + {"use_local_cache": False, "event_key_jmespath": "body"}, ], indirect=True, ) @@ -1107,8 +1074,7 @@ def test_default_no_raise_on_missing_idempotency_key( @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "event_key_jmespath": "[body, x]", "expires_in_progress": True}, - {"use_local_cache": False, "event_key_jmespath": "[body, x]", "expires_in_progress": False}, + {"use_local_cache": False, "event_key_jmespath": "[body, x]"}, ], indirect=True, ) @@ -1135,12 +1101,6 @@ def test_raise_on_no_idempotency_key( { "use_local_cache": False, "event_key_jmespath": "[requestContext.authorizer.claims.sub, powertools_json(body).id]", - "expires_in_progress": False, - }, - { - "use_local_cache": False, - "event_key_jmespath": "[requestContext.authorizer.claims.sub, powertools_json(body).id]", - "expires_in_progress": True, }, ], indirect=True, @@ -1397,8 +1357,7 @@ def dummy_handler(event, context): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": True, "expires_in_progress": True}, - {"use_local_cache": True, "expires_in_progress": False}, + {"use_local_cache": True}, ], indirect=True, ) @@ -1521,8 +1480,7 @@ def collect_payment(payment: Payment): @pytest.mark.parametrize( "idempotency_config", [ - {"use_local_cache": False, "expires_in_progress": True}, - {"use_local_cache": False, "expires_in_progress": False}, + {"use_local_cache": False}, ], indirect=True, ) diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index 383a5298bf6..e8db8f5dd0a 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -3,7 +3,6 @@ from botocore import stub -from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from tests.functional.utils import json_serialize @@ -14,61 +13,53 @@ def hash_idempotency_key(data: Any): def build_idempotency_put_item_stub( data: Dict, - config: IdempotencyConfig, function_name: str = "test-func", handler_name: str = "lambda_handler", ) -> Dict: idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" - params = { - "ConditionExpression": ("attribute_not_exists(#id) OR #now < :now"), + return { + "ConditionExpression": ( + "attribute_not_exists(#id) OR #now < :now OR " + "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + ), "ExpressionAttributeNames": { "#id": "id", "#now": "expiration", "#status": "status", + "#in_progress_expiry": "in_progress_expiration", }, - "ExpressionAttributeValues": {":now": stub.ANY}, + "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } - if config.expires_in_progress: - params[ - "ConditionExpression" - ] += " OR (attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" - params["ExpressionAttributeValues"][":inprogress"] = "INPROGRESS" - - return params - def build_idempotency_update_item_stub( data: Dict, handler_response: Dict, - config: IdempotencyConfig, function_name: str = "test-func", handler_name: str = "lambda_handler", ) -> Dict: idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" serialized_lambda_response = json_serialize(handler_response) - params = { + return { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", "#status": "status", + "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", + ":in_progress_expiry": stub.ANY, }, "Key": {"id": idempotency_key_hash}, "TableName": "TEST_TABLE", - "UpdateExpression": ("SET #response_data = :response_data, " "#expiry = :expiry, #status = :status"), + "UpdateExpression": ( + "SET #response_data = :response_data, " + "#expiry = :expiry, #status = :status, " + "#in_progress_expiry = :in_progress_expiry" + ), } - - if config.expires_in_progress: - params["ExpressionAttributeNames"]["#in_progress_expiry"] = "in_progress_expiration" - params["ExpressionAttributeValues"][":in_progress_expiry"] = stub.ANY - params["UpdateExpression"] += ", #in_progress_expiry = :in_progress_expiry" - - return params From 61f94b374c36eac34bf5d29600122a87e3c3299d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 16:16:19 +0200 Subject: [PATCH 12/26] chore(idempotency): remove more of the old code --- .../utilities/idempotency/base.py | 1 - docs/utilities/idempotency.md | 2 +- tests/functional/idempotency/conftest.py | 10 +- .../idempotency/test_idempotency.py | 185 +++--------------- 4 files changed, 26 insertions(+), 172 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index ed9fb8b6dfd..1929057a2c0 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -74,7 +74,6 @@ def __init__( self.data = deepcopy(_prepare_data(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs - self.config = config persistence_store.configure(config, self.function.__name__) self.persistence_store = persistence_store diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 78089051ff0..9f03a7f9487 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -22,7 +22,7 @@ times with the same parameters**. This makes idempotent operations safe to retry * Ensure Lambda handler returns the same result when called with the same payload * Select a subset of the event as the idempotency key using JMESPath expressions * Set a time window in which records with the same payload should be considered duplicates -* Optionally expires in-progress executions after the Lambda handler timeout +* Expires in-progress executions when the Lambda handler times out ## Getting started diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index bb58597af0d..2104ac23b26 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -76,7 +76,7 @@ def default_jmespath(): @pytest.fixture def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key): - params = { + return { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", @@ -97,14 +97,12 @@ def expected_params_update_item(serialized_lambda_response, hashed_idempotency_k ), } - return params - @pytest.fixture def expected_params_update_item_with_validation( - serialized_lambda_response, hashed_idempotency_key, hashed_validation_key, idempotency_config + serialized_lambda_response, hashed_idempotency_key, hashed_validation_key ): - params = { + return { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", @@ -129,8 +127,6 @@ def expected_params_update_item_with_validation( ), } - return params - @pytest.fixture def expected_params_put_item(hashed_idempotency_key): diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 9b1dc2b048e..1de4232340f 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -43,14 +43,7 @@ def get_dataclasses_lib(): # Using parametrize to run test twice, with two separate instances of persistence store. One instance with caching # enabled, and one without. -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_already_completed( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -95,14 +88,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -151,13 +137,7 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress_with_cache( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -222,14 +202,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -260,13 +233,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution_cached( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -312,13 +279,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": True, "event_key_jmespath": "body"}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True, "event_key_jmespath": "body"}], indirect=True) def test_idempotent_lambda_first_execution_event_mutation( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -356,14 +317,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -396,14 +350,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_exception( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -491,14 +438,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired_during_request( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -554,14 +494,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_deleting( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -593,14 +526,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_updating( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -632,14 +558,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_getting( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -706,12 +625,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "config_without_jmespath", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, + "config_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_with_validator_util( config_without_jmespath: IdempotencyConfig, @@ -762,14 +676,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expires_in_progress_before_expire( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -821,14 +728,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expires_in_progress_after_expire( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -873,14 +773,7 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expires_in_progress_unavailable_remaining_time( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, @@ -943,13 +836,7 @@ def test_data_record_json_to_dict_mapping_when_response_data_none(): assert response_data is None -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_in_progress_never_saved_to_cache( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): @@ -965,13 +852,7 @@ def test_in_progress_never_saved_to_cache( assert persistence_store._cache.get("key") is None -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): # GIVEN a persistence_store with use_local_cache = False persistence_store.configure(idempotency_config) @@ -991,13 +872,7 @@ def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_ assert not hasattr("persistence_store", "_cache") -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_delete_from_cache_when_empty( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): @@ -1048,11 +923,7 @@ def test_is_missing_idempotency_key(): @pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False, "event_key_jmespath": "body"}, - ], - indirect=True, + "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True ) def test_default_no_raise_on_missing_idempotency_key( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer @@ -1354,13 +1225,7 @@ def dummy_handler(event, context): assert len(persistence_store.table.method_calls) == 0 -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": True}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_function_duplicates( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): @@ -1477,13 +1342,7 @@ def collect_payment(payment: Payment): assert result == payment.transaction_id -@pytest.mark.parametrize( - "idempotency_config", - [ - {"use_local_cache": False}, - ], - indirect=True, -) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_idempotent_lambda_compound_already_completed( idempotency_config: IdempotencyConfig, persistence_store_compound: DynamoDBPersistenceLayer, From fa80aedfe03b66d60cc344a870a37eea8aea74c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 16:19:25 +0200 Subject: [PATCH 13/26] chore(docs): remove bad comment --- tests/functional/idempotency/test_idempotency.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 1de4232340f..c6cb9af07b8 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -686,11 +686,6 @@ def test_idempotent_lambda_expires_in_progress_before_expire( hashed_idempotency_key, lambda_context, ): - """ - Test idempotent decorator when expires_in_progress is on and the event is still in progress, before the - lambda expiration window. - """ - stubber = stub.Stubber(persistence_store.table.meta.client) stubber.add_client_error("put_item", "ConditionalCheckFailedException") From c706b0cc2fac18c2e30cbde7cc344c9351bfb7b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 16:35:37 +0200 Subject: [PATCH 14/26] feat(idempotench): add mechanism to register lambda context --- .../utilities/idempotency/base.py | 5 +++ .../utilities/idempotency/config.py | 6 ++++ docs/utilities/idempotency.md | 31 ++++++++++++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 1929057a2c0..59ace30b1be 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -74,6 +74,7 @@ def __init__( self.data = deepcopy(_prepare_data(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs + self.config = config persistence_store.configure(config, self.function.__name__) self.persistence_store = persistence_store @@ -129,6 +130,10 @@ def _get_remaining_time_in_millis(self) -> Optional[int]: Remaining time in millis, or None if the remaining time cannot be determined. """ + # Look to see if we have stored a Lambda Context + if self.config.lambda_context is not None: + self.config.lambda_context.get_remainig_time_in_millis() + # Look into fn_args to see if we have a lambda context if self.fn_args and len(self.fn_args) == 2 and getattr(self.fn_args[1], "get_remaining_time_in_millis", None): return self.fn_args[1].get_remaining_time_in_millis() diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py index 06468cc74a7..1b657c6aa9d 100644 --- a/aws_lambda_powertools/utilities/idempotency/config.py +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -1,5 +1,7 @@ from typing import Dict, Optional +from aws_lambda_powertools.utilities.typing import LambdaContext + class IdempotencyConfig: def __init__( @@ -41,3 +43,7 @@ def __init__( self.use_local_cache = use_local_cache self.local_cache_max_items = local_cache_max_items self.hash_function = hash_function + self.lambda_context = None + + def register_lambda_context(self, lambda_context: LambdaContext): + self.lambda_context = lambda_context diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 9f03a7f9487..0d059e8002b 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -400,11 +400,6 @@ after this timestamp, and the record is still marked `INPROGRESS`, we execute th already expired. This means that if an invocation expired during execution, it will be quickly executed again on the next retry. -???+ info "Info: Calculating the remaining available time" - For now this only works with the `idempotent` decorator. At the moment we - don't have access to the Lambda context when using the - `idempotent_function` so enabling this option is a no-op in that scenario. -
```mermaid sequenceDiagram @@ -439,6 +434,32 @@ sequenceDiagram Idempotent sequence for Lambda timeouts
+???+ info "Info: Calculating the remaining available time" + When using the `idempotent` decorator, we captura and calculate the remaining available time for you. + However, when using the `idempotent_function`, the functionality doesn't work out of the box. You'll + need to register the Lambda context on your handler: + +```python hl_lines="8 16" title="Registering the Lambda context" +from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord +from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, idempotent_function +) + +persistence_layer = DynamoDBPersistenceLayer(table_name="...") + +config = IdempotencyConfig() + +@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) +def record_handler(record: SQSRecord): + return {"message": record["body"]} + + +def lambda_handler(event, context): + config.register_lambda_context(context) + + return record_handler(event) +``` + ### Handling exceptions If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. From 1a72214a3e1edd39233a3d299c144fef53c15e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 27 Jul 2022 16:42:42 +0200 Subject: [PATCH 15/26] fix(idempotency): typo --- aws_lambda_powertools/utilities/idempotency/base.py | 2 +- aws_lambda_powertools/utilities/idempotency/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 59ace30b1be..03ae8d1acb5 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -132,7 +132,7 @@ def _get_remaining_time_in_millis(self) -> Optional[int]: # Look to see if we have stored a Lambda Context if self.config.lambda_context is not None: - self.config.lambda_context.get_remainig_time_in_millis() + self.config.lambda_context.get_remaining_time_in_millis() # Look into fn_args to see if we have a lambda context if self.fn_args and len(self.fn_args) == 2 and getattr(self.fn_args[1], "get_remaining_time_in_millis", None): diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py index 1b657c6aa9d..eacc5819ee7 100644 --- a/aws_lambda_powertools/utilities/idempotency/config.py +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -43,7 +43,7 @@ def __init__( self.use_local_cache = use_local_cache self.local_cache_max_items = local_cache_max_items self.hash_function = hash_function - self.lambda_context = None + self.lambda_context: Optional[LambdaContext] = None def register_lambda_context(self, lambda_context: LambdaContext): self.lambda_context = lambda_context From 0ce0bb2b0ed2d3141bf26a6ecd27b606491868ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 28 Jul 2022 14:29:32 +0200 Subject: [PATCH 16/26] fix(idempotency): capture the lambda context automatically --- .../utilities/idempotency/base.py | 7 +--- .../utilities/idempotency/idempotency.py | 2 ++ docs/utilities/idempotency.md | 8 ++--- tests/functional/idempotency/conftest.py | 26 ++++++++++----- .../idempotency/test_idempotency.py | 33 +++++-------------- tests/functional/idempotency/utils.py | 7 +++- 6 files changed, 39 insertions(+), 44 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 03ae8d1acb5..2980904a72f 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -130,13 +130,8 @@ def _get_remaining_time_in_millis(self) -> Optional[int]: Remaining time in millis, or None if the remaining time cannot be determined. """ - # Look to see if we have stored a Lambda Context if self.config.lambda_context is not None: - self.config.lambda_context.get_remaining_time_in_millis() - - # Look into fn_args to see if we have a lambda context - if self.fn_args and len(self.fn_args) == 2 and getattr(self.fn_args[1], "get_remaining_time_in_millis", None): - return self.fn_args[1].get_remaining_time_in_millis() + return self.config.lambda_context.get_remaining_time_in_millis() return None diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 4a7d8e71e1d..646fd68558f 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -62,6 +62,8 @@ def idempotent( return handler(event, context) config = config or IdempotencyConfig() + config.register_lambda_context(context) + args = event, context idempotency_handler = IdempotencyHandler( function=handler, diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 0d059e8002b..48b159e7033 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -392,11 +392,11 @@ In cases where the [Lambda invocation expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools doesn't have the chance to set the idempotency record to `EXPIRED`. This means that the record would normally have been locked until `expire_seconds` have -passsed. +passed. However, when Powertools has access to the Lambda invocation context, we are able to calculate the remaining available time for the invocation, and save it on the idempotency record. This way, if a second invocation happens -after this timestamp, and the record is still marked `INPROGRESS`, we execute the inovcation again as if it was +after this timestamp, and the record is still marked `INPROGRESS`, we execute the invocation again as if it was already expired. This means that if an invocation expired during execution, it will be quickly executed again on the next retry. @@ -435,9 +435,9 @@ sequenceDiagram ???+ info "Info: Calculating the remaining available time" - When using the `idempotent` decorator, we captura and calculate the remaining available time for you. + When using the `idempotent` decorator, we capture and calculate the remaining available time for you. However, when using the `idempotent_function`, the functionality doesn't work out of the box. You'll - need to register the Lambda context on your handler: + need to register the Lambda context in your handler: ```python hl_lines="8 16" title="Registering the Lambda context" from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 2104ac23b26..848cbbf5fa3 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -1,6 +1,5 @@ import datetime import json -from collections import namedtuple from decimal import Decimal from unittest import mock @@ -32,14 +31,17 @@ def lambda_apigw_event(): @pytest.fixture def lambda_context(): - lambda_context = { - "function_name": "test-func", - "memory_limit_in_mb": 128, - "invoked_function_arn": "arn:aws:lambda:eu-west-1:809313241234:function:test-func", - "aws_request_id": "52fdfc07-2182-154f-163f-5f0f9a621d72", - } + class LambdaContext: + def __init__(self): + self.function_name = "test-func" + self.memory_limit_in_mb = 128 + self.invoked_function_arn = "arn:aws:lambda:eu-west-1:809313241234:function:test-func" + self.aws_request_id = "52fdfc07-2182-154f-163f-5f0f9a621d72" + + def get_remaining_time_in_millis(self) -> int: + return 1000 - return namedtuple("LambdaContext", lambda_context.keys())(*lambda_context.values()) + return LambdaContext() @pytest.fixture @@ -142,7 +144,12 @@ def expected_params_put_item(hashed_idempotency_key): "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, - "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, + "Item": { + "expiration": stub.ANY, + "id": hashed_idempotency_key, + "status": "INPROGRESS", + "in_progress_expiration": stub.ANY, + }, "TableName": "TEST_TABLE", } @@ -163,6 +170,7 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": { "expiration": stub.ANY, + "in_progress_expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS", "validation": hashed_validation_key, diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index c6cb9af07b8..21ef4038da8 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -768,37 +768,22 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) -def test_idempotent_lambda_expires_in_progress_unavailable_remaining_time( - idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, - lambda_apigw_event, - lambda_response, - lambda_context, - expected_params_put_item, - expected_params_update_item, -): - stubber = stub.Stubber(persistence_store.table.meta.client) - - ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item) - stubber.add_response("update_item", ddb_response, expected_params_update_item) - - stubber.activate() +def test_idempotent_lambda_expires_in_progress_unavailable_remaining_time(): + mock_event = {"data": "value"} + idempotency_key = "test-func.function#" + hash_idempotency_key(mock_event) + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + expected_result = {"message": "Foo"} - @idempotent(config=idempotency_config, persistence_store=persistence_store) - def lambda_handler(event, context): - return lambda_response + @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") + def function(record): + return expected_result with warnings.catch_warnings(record=True) as w: warnings.simplefilter("default") - lambda_handler(lambda_apigw_event, lambda_context) + function(record=mock_event) assert len(w) == 1 assert str(w[-1].message) == "Expires in progress is enabled but we couldn't determine the remaining time left" - stubber.assert_no_pending_responses() - stubber.deactivate() - def test_data_record_invalid_status_value(): data_record = DataRecord("key", status="UNSUPPORTED_STATUS") diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index e8db8f5dd0a..2a5b7e95920 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -29,7 +29,12 @@ def build_idempotency_put_item_stub( "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, - "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"}, + "Item": { + "expiration": stub.ANY, + "id": idempotency_key_hash, + "status": "INPROGRESS", + "in_progress_expiration": stub.ANY, + }, "TableName": "TEST_TABLE", } From a2b6a3418b875ea08850d61550b04caa330108bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 13:55:44 +0200 Subject: [PATCH 17/26] chore(idempotency): addressed review comments --- .../utilities/idempotency/base.py | 5 ++- .../utilities/idempotency/config.py | 5 ++- .../utilities/idempotency/persistence/base.py | 8 +--- .../idempotency/persistence/dynamodb.py | 43 ++++++++++++++++--- tests/functional/idempotency/conftest.py | 16 +++---- tests/functional/idempotency/utils.py | 8 ++-- 6 files changed, 57 insertions(+), 28 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 2980904a72f..9b8bddb52a9 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -121,8 +121,9 @@ def _get_remaining_time_in_millis(self) -> Optional[int]: """ Tries to determine the remaining time available for the current lambda invocation. - Currently, it only works if the idempotent handler decorator is used, since we need to acess the lambda context. - However, this could be improved if we start storing the lambda context globally during the invocation. + This only works if the idempotent handler decorator is used, since we need to access the lambda context. + However, this could be improved if we start storing the lambda context globally during the invocation. One + way to do this is to register the lambda context when configuring the IdempotencyConfig object. Returns ------- diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py index eacc5819ee7..07211537208 100644 --- a/aws_lambda_powertools/utilities/idempotency/config.py +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -14,6 +14,7 @@ def __init__( use_local_cache: bool = False, local_cache_max_items: int = 256, hash_function: str = "md5", + lambda_context: Optional[LambdaContext] = None, ): """ Initialize the base persistence layer @@ -34,6 +35,8 @@ def __init__( Max number of items to store in local cache, by default 1024 hash_function: str, optional Function to use for calculating hashes, by default md5. + lambda_context: LambdaContext, optional + Lambda Context containing information about the invocation, function and execution environment. """ self.event_key_jmespath = event_key_jmespath self.payload_validation_jmespath = payload_validation_jmespath @@ -43,7 +46,7 @@ def __init__( self.use_local_cache = use_local_cache self.local_cache_max_items = local_cache_max_items self.hash_function = hash_function - self.lambda_context: Optional[LambdaContext] = None + self.lambda_context: Optional[LambdaContext] = lambda_context def register_lambda_context(self, lambda_context: LambdaContext): self.lambda_context = lambda_context diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 920f93bdae7..21dcac65b37 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -8,7 +8,6 @@ import os import warnings from abc import ABC, abstractmethod -from math import ceil from types import MappingProxyType from typing import Any, Dict, Optional @@ -355,12 +354,7 @@ def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Option now = datetime.datetime.now() period = datetime.timedelta(milliseconds=remaining_time_in_millis) - # It's very important to use math.ceil here. Otherwise, we might return an integer that will be smaller - # than the current time in milliseconds, due to rounding. This will create a scenario where the record - # looks already expired in the store, but the invocation is still running. - timestamp = ceil((now + period).timestamp()) - - data_record.in_progress_expiry_timestamp = timestamp + data_record.in_progress_expiry_timestamp = int((now + period).timestamp() * 1000) else: warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left") diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index bd253b4a3e5..76419f8d482 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -168,19 +168,50 @@ def _put_record(self, data_record: DataRecord) -> None: try: logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}") + # | LOCKED | RETRY if status = "INPROGRESS" | RETRY + # |----------------|-------------------------------------------------------|-------------> .... (time) + # | Lambda Idempotency Record + # | Timeout Timeout + # | (in_progress_expiry) (expiry) + + # Conditions to successfully save a record: + + # The idempotency key does not exist: + # - first time that this invocation key is used + # - previous invocation with the same key was deleted due to TTL + idempotency_key_not_exist = "attribute_not_exists(#id)" + + # The idempotency record exists but it's expired: + idempotency_expiry_expired = "#expiry < :now" + + # The status of the record is "INPROGRESS", there is an in-progress expiry timestamp, but it's expired + inprogress_expiry_expired = " AND ".join( + [ + "#status = :inprogress", + "attribute_exists(#in_progress_expiry)", + "#in_progress_expiry < :now_in_millis", + ] + ) + inprogress_expiry_expired = f"({inprogress_expiry_expired})" + + condition_expression = " OR ".join( + [idempotency_key_not_exist, idempotency_expiry_expired, inprogress_expiry_expired] + ) + self.table.put_item( Item=item, - ConditionExpression=( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" - ), + ConditionExpression=condition_expression, ExpressionAttributeNames={ "#id": self.key_attr, - "#now": self.expiry_attr, + "#expiry": self.expiry_attr, "#in_progress_expiry": self.in_progress_expiry_attr, "#status": self.status_attr, }, - ExpressionAttributeValues={":now": int(now.timestamp()), ":inprogress": STATUS_CONSTANTS["INPROGRESS"]}, + ExpressionAttributeValues={ + ":now": int(now.timestamp()), + ":now_in_millis": int(now.timestamp() * 1000), + ":inprogress": STATUS_CONSTANTS["INPROGRESS"], + }, ) except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 848cbbf5fa3..9731984201a 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -134,16 +134,16 @@ def expected_params_update_item_with_validation( def expected_params_put_item(hashed_idempotency_key): return { "ConditionExpression": ( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + "attribute_not_exists(#id) OR #expiry < :now OR " + "(#status = :inprogress AND attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now_in_millis)" ), "ExpressionAttributeNames": { "#id": "id", - "#now": "expiration", + "#expiry": "expiration", "#status": "status", "#in_progress_expiry": "in_progress_expiration", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY, ":now_in_millis": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": { "expiration": stub.ANY, "id": hashed_idempotency_key, @@ -158,16 +158,16 @@ def expected_params_put_item(hashed_idempotency_key): def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key): return { "ConditionExpression": ( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + "attribute_not_exists(#id) OR #expiry < :now OR " + "(#status = :inprogress AND attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now_in_millis)" ), "ExpressionAttributeNames": { "#id": "id", - "#now": "expiration", + "#expiry": "expiration", "#status": "status", "#in_progress_expiry": "in_progress_expiration", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY, ":now_in_millis": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": { "expiration": stub.ANY, "in_progress_expiration": stub.ANY, diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index 2a5b7e95920..833691be65a 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -19,16 +19,16 @@ def build_idempotency_put_item_stub( idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}" return { "ConditionExpression": ( - "attribute_not_exists(#id) OR #now < :now OR " - "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)" + "attribute_not_exists(#id) OR #expiry < :now OR " + "(#status = :inprogress AND attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now_in_millis)" ), "ExpressionAttributeNames": { "#id": "id", - "#now": "expiration", + "#expiry": "expiration", "#status": "status", "#in_progress_expiry": "in_progress_expiration", }, - "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"}, + "ExpressionAttributeValues": {":now": stub.ANY, ":now_in_millis": stub.ANY, ":inprogress": "INPROGRESS"}, "Item": { "expiration": stub.ANY, "id": idempotency_key_hash, From 228a76d4ffa7d804a387562fe5a3befee9dedc9e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 29 Jul 2022 14:54:07 +0200 Subject: [PATCH 18/26] docs(idempotency): include register_lambda_context in doc snippets Signed-off-by: heitorlessa --- docs/utilities/idempotency.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 48b159e7033..aff5f9b08d5 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -162,6 +162,7 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo def lambda_handler(event, context): # `data` parameter must be called as a keyword argument to work dummy("hello", "universe", data="test") + config.register_lambda_context(context) # see Lambda timeouts section return processor.response() ``` @@ -198,7 +199,7 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo === "dataclass_sample.py" - ```python hl_lines="3-4 23 32" + ```python hl_lines="3-4 23 33" from dataclasses import dataclass from aws_lambda_powertools.utilities.idempotency import ( @@ -225,17 +226,18 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo def process_order(order: Order): return f"processed order {order.order_id}" + def lambda_handler(event, context): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id="fake-id") - order_item = OrderItem(sku="fake", description="sample") - order = Order(item=order_item, order_id="fake-id") - - # `order` parameter must be called as a keyword argument to work - process_order(order=order) + # `order` parameter must be called as a keyword argument to work + process_order(order=order) ``` === "parser_pydantic_sample.py" - ```python hl_lines="1-2 22 31" + ```python hl_lines="1-2 22 32" from aws_lambda_powertools.utilities.idempotency import ( DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) from aws_lambda_powertools.utilities.parser import BaseModel @@ -261,12 +263,13 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo def process_order(order: Order): return f"processed order {order.order_id}" + def lambda_handler(event, context): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id="fake-id") - order_item = OrderItem(sku="fake", description="sample") - order = Order(item=order_item, order_id="fake-id") - - # `order` parameter must be called as a keyword argument to work - process_order(order=order) + # `order` parameter must be called as a keyword argument to work + process_order(order=order) ``` ### Choosing a payload subset for idempotency From 3cb7411974dc20309f5317bb0059af83faa39188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 15:03:33 +0200 Subject: [PATCH 19/26] chore(idempotency): added tests for handle_for_status --- .../utilities/idempotency/base.py | 2 +- .../idempotency/test_idempotency.py | 64 +++++++++++++++++-- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 9b8bddb52a9..ddd054daa14 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -191,7 +191,7 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] if data_record.status == STATUS_CONSTANTS["INPROGRESS"]: if data_record.in_progress_expiry_timestamp is not None and data_record.in_progress_expiry_timestamp < int( - datetime.datetime.now().timestamp() + datetime.datetime.now().timestamp() * 1000 ): raise IdempotencyInconsistentStateError( "item should have been expired in-progress because it already time-outed." diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 21ef4038da8..3da04bf6313 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -12,7 +12,7 @@ from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEventV2, event_source from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig -from aws_lambda_powertools.utilities.idempotency.base import MAX_RETRIES, _prepare_data +from aws_lambda_powertools.utilities.idempotency.base import MAX_RETRIES, IdempotencyHandler, _prepare_data from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -692,7 +692,7 @@ def test_idempotent_lambda_expires_in_progress_before_expire( now = datetime.datetime.now() period = datetime.timedelta(seconds=5) - timestamp_expires_in_progress = str(int((now + period).timestamp())) + timestamp_expires_in_progress = int((now + period).timestamp() * 1000) expected_params_get_item = { "TableName": TABLE_NAME, @@ -703,7 +703,7 @@ def test_idempotent_lambda_expires_in_progress_before_expire( "Item": { "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_future}, - "in_progress_expiration": {"N": timestamp_expires_in_progress}, + "in_progress_expiration": {"N": str(timestamp_expires_in_progress)}, "data": {"S": '{"message": "test", "statusCode": 200'}, "status": {"S": "INPROGRESS"}, } @@ -748,7 +748,7 @@ def test_idempotent_lambda_expires_in_progress_after_expire( "Item": { "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_future}, - "in_progress_expiration": {"N": str(int(one_second_ago.timestamp()))}, + "in_progress_expiration": {"N": str(int(one_second_ago.timestamp() * 1000))}, "data": {"S": '{"message": "test", "statusCode": 200'}, "status": {"S": "INPROGRESS"}, } @@ -816,6 +816,62 @@ def test_data_record_json_to_dict_mapping_when_response_data_none(): assert response_data is None +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_handler_for_status_expired_data_record( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): + idempotency_handler = IdempotencyHandler( + function=lambda a: a, + function_payload={}, + config=idempotency_config, + persistence_store=persistence_store, + ) + data_record = DataRecord("key", status="EXPIRED", response_data=None) + + with pytest.raises(IdempotencyInconsistentStateError): + idempotency_handler._handle_for_status(data_record) + + +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_handler_for_status_inprogress_data_record_inconsistent( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): + idempotency_handler = IdempotencyHandler( + function=lambda a: a, + function_payload={}, + config=idempotency_config, + persistence_store=persistence_store, + ) + + now = datetime.datetime.now() + period = datetime.timedelta(milliseconds=100) + timestamp = int((now - period).timestamp() * 1000) + data_record = DataRecord("key", in_progress_expiry_timestamp=timestamp, status="INPROGRESS", response_data=None) + + with pytest.raises(IdempotencyInconsistentStateError): + idempotency_handler._handle_for_status(data_record) + + +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_handler_for_status_inprogress_data_record_consistent( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): + idempotency_handler = IdempotencyHandler( + function=lambda a: a, + function_payload={}, + config=idempotency_config, + persistence_store=persistence_store, + ) + + now = datetime.datetime.now() + period = datetime.timedelta(milliseconds=100) + timestamp = int((now + period).timestamp() * 1000) + data_record = DataRecord("key", in_progress_expiry_timestamp=timestamp, status="INPROGRESS", response_data=None) + + with pytest.raises(IdempotencyAlreadyInProgressError): + idempotency_handler._handle_for_status(data_record) + + @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_in_progress_never_saved_to_cache( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer From 5c09a5aed5156d8a4d118e204a91439611fc7acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 15:06:31 +0200 Subject: [PATCH 20/26] chore(docs): add documentation to method --- aws_lambda_powertools/utilities/idempotency/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py index 07211537208..e78f339fdc9 100644 --- a/aws_lambda_powertools/utilities/idempotency/config.py +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -49,4 +49,5 @@ def __init__( self.lambda_context: Optional[LambdaContext] = lambda_context def register_lambda_context(self, lambda_context: LambdaContext): + """Captures the Lambda context, to calculate the remaining time before the invocation times out""" self.lambda_context = lambda_context From 2e1afd34cefdc82c699ee6b7a1c176091e3ef14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 15:09:57 +0200 Subject: [PATCH 21/26] chore(idempotency): address comments --- .../utilities/idempotency/persistence/base.py | 8 ++++++-- tests/functional/idempotency/test_idempotency.py | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 21dcac65b37..a87980d7fe0 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -353,10 +353,14 @@ def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Option if remaining_time_in_millis: now = datetime.datetime.now() period = datetime.timedelta(milliseconds=remaining_time_in_millis) + timestamp = (now + period).timestamp() - data_record.in_progress_expiry_timestamp = int((now + period).timestamp() * 1000) + data_record.in_progress_expiry_timestamp = int(timestamp * 1000) else: - warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left") + warnings.warn( + "Couldn't determine the remaining time left. " + "Did you call register_lambda_context on IdempotencyConfig?" + ) logger.debug(f"Saving in progress record for idempotency key: {data_record.idempotency_key}") diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 3da04bf6313..97a9166efa0 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -782,7 +782,10 @@ def function(record): warnings.simplefilter("default") function(record=mock_event) assert len(w) == 1 - assert str(w[-1].message) == "Expires in progress is enabled but we couldn't determine the remaining time left" + assert ( + str(w[-1].message) + == "Couldn't determine the remaining time left. Did you call register_lambda_context on IdempotencyConfig?" + ) def test_data_record_invalid_status_value(): From 0e9dfd4ef523933d211b203412d080a3fbe4ef46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 15:11:42 +0200 Subject: [PATCH 22/26] chore(idempotency): simplified strings --- .../utilities/idempotency/persistence/dynamodb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 76419f8d482..a8b2e4c911d 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -192,10 +192,9 @@ def _put_record(self, data_record: DataRecord) -> None: "#in_progress_expiry < :now_in_millis", ] ) - inprogress_expiry_expired = f"({inprogress_expiry_expired})" - condition_expression = " OR ".join( - [idempotency_key_not_exist, idempotency_expiry_expired, inprogress_expiry_expired] + condition_expression = ( + f"{idempotency_key_not_exist} OR {idempotency_expiry_expired} OR ({inprogress_expiry_expired})" ) self.table.put_item( From 81ed53d213e02f1574615f0e85717b42960a10bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 15:13:50 +0200 Subject: [PATCH 23/26] chore(documentation): addressed comments --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index aff5f9b08d5..683e8f4f095 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -22,7 +22,7 @@ times with the same parameters**. This makes idempotent operations safe to retry * Ensure Lambda handler returns the same result when called with the same payload * Select a subset of the event as the idempotency key using JMESPath expressions * Set a time window in which records with the same payload should be considered duplicates -* Expires in-progress executions when the Lambda handler times out +* Expires in-progress executions if the Lambda function times out halfway through ## Getting started From 66b62a6c4849217f13034a2c264a4ec57ac37415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Fri, 29 Jul 2022 15:18:04 +0200 Subject: [PATCH 24/26] chore(idempotency): no need to update expire on update --- .../utilities/idempotency/persistence/dynamodb.py | 9 ++------- tests/functional/idempotency/conftest.py | 10 +--------- tests/functional/idempotency/utils.py | 8 +------- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index a8b2e4c911d..90cbd853e8a 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -218,21 +218,16 @@ def _put_record(self, data_record: DataRecord) -> None: def _update_record(self, data_record: DataRecord): logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") - update_expression = ( - "SET #response_data = :response_data, #expiry = :expiry, " - "#status = :status, #in_progress_expiry = :in_progress_expiry" - ) + update_expression = "SET #response_data = :response_data, #expiry = :expiry, " "#status = :status" expression_attr_values = { ":expiry": data_record.expiry_timestamp, ":response_data": data_record.response_data, ":status": data_record.status, - ":in_progress_expiry": data_record.in_progress_expiry_timestamp, } expression_attr_names = { - "#response_data": self.data_attr, "#expiry": self.expiry_attr, + "#response_data": self.data_attr, "#status": self.status_attr, - "#in_progress_expiry": self.in_progress_expiry_attr, } if self.payload_validation_enabled: diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 9731984201a..b5cf79727b1 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -83,20 +83,15 @@ def expected_params_update_item(serialized_lambda_response, hashed_idempotency_k "#expiry": "expiration", "#response_data": "data", "#status": "status", - "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", - ":in_progress_expiry": stub.ANY, }, "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", - "UpdateExpression": ( - "SET #response_data = :response_data, " - "#expiry = :expiry, #status = :status, #in_progress_expiry = :in_progress_expiry" - ), + "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", } @@ -110,21 +105,18 @@ def expected_params_update_item_with_validation( "#response_data": "data", "#status": "status", "#validation_key": "validation", - "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", ":validation_key": hashed_validation_key, - ":in_progress_expiry": stub.ANY, }, "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", "UpdateExpression": ( "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status, " - "#in_progress_expiry = :in_progress_expiry, " "#validation_key = :validation_key" ), } diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index 833691be65a..797b696aba4 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -52,19 +52,13 @@ def build_idempotency_update_item_stub( "#expiry": "expiration", "#response_data": "data", "#status": "status", - "#in_progress_expiry": "in_progress_expiration", }, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", - ":in_progress_expiry": stub.ANY, }, "Key": {"id": idempotency_key_hash}, "TableName": "TEST_TABLE", - "UpdateExpression": ( - "SET #response_data = :response_data, " - "#expiry = :expiry, #status = :status, " - "#in_progress_expiry = :in_progress_expiry" - ), + "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", } From 84ced8e1b9c0191b1d9c15318c5258b419687a16 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 29 Jul 2022 15:21:54 +0200 Subject: [PATCH 25/26] docs(idempotency): reorder wording, banners to emphasize the need, and split diag section Signed-off-by: heitorlessa --- docs/utilities/idempotency.md | 132 +++++++++++++++++----------------- 1 file changed, 67 insertions(+), 65 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index aff5f9b08d5..15cc169b4be 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -36,10 +36,10 @@ As of now, Amazon DynamoDB is the only supported persistent storage layer, so yo If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: -Configuration | Value | Notes -------------------------------------------------- | ------------------------------------------------- | ------------------------------------------------- -Partition key | `id` | -TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console +| Configuration | Value | Notes | +| ------------------ | ------------ | ----------------------------------------------------------------------------------- | +| Partition key | `id` | +| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console | ???+ tip "Tip: You can share a single state table for all functions" You can reuse the same DynamoDB table to store idempotency state. We add your `function_name` in addition to the idempotency key as a hash key. @@ -391,17 +391,45 @@ The client was successful in receiving the result after the retry. Since the Lam #### Lambda timeouts -In cases where the [Lambda invocation -expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), -Powertools doesn't have the chance to set the idempotency record to `EXPIRED`. -This means that the record would normally have been locked until `expire_seconds` have -passed. +???+ note + This is automatically done when you decorate your Lambda handler with [@idempotent decorator](#idempotent-decorator). + +To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools calculates and includes the remaining invocation available time as part of the idempotency record. + +???+ example + If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). + + This means that if an invocation expired during execution, it will be quickly executed again on the next retry. + +???+ important + If you are only using the [@idempotent_function decorator](#idempotentfunction-decorator) to guard isolated parts of your code, you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + +Here is an example on how you register the Lambda context in your handler: + +```python hl_lines="8 16" title="Registering the Lambda context" +from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord +from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, idempotent_function +) + +persistence_layer = DynamoDBPersistenceLayer(table_name="...") + +config = IdempotencyConfig() + +@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) +def record_handler(record: SQSRecord): + return {"message": record["body"]} + + +def lambda_handler(event, context): + config.register_lambda_context(context) + + return record_handler(event) +``` -However, when Powertools has access to the Lambda invocation context, we are able to calculate the remaining -available time for the invocation, and save it on the idempotency record. This way, if a second invocation happens -after this timestamp, and the record is still marked `INPROGRESS`, we execute the invocation again as if it was -already expired. This means that if an invocation expired during execution, it will be quickly executed again on the -next retry. +#### Lambda timeout sequence diagram + +This sequence diagram shows an example flow of what happens if a Lambda function times out:
```mermaid @@ -437,32 +465,6 @@ sequenceDiagram Idempotent sequence for Lambda timeouts
-???+ info "Info: Calculating the remaining available time" - When using the `idempotent` decorator, we capture and calculate the remaining available time for you. - However, when using the `idempotent_function`, the functionality doesn't work out of the box. You'll - need to register the Lambda context in your handler: - -```python hl_lines="8 16" title="Registering the Lambda context" -from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, idempotent_function -) - -persistence_layer = DynamoDBPersistenceLayer(table_name="...") - -config = IdempotencyConfig() - -@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) -def record_handler(record: SQSRecord): - return {"message": record["body"]} - - -def lambda_handler(event, context): - config.register_lambda_context(context) - - return record_handler(event) -``` - ### Handling exceptions If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. @@ -536,17 +538,17 @@ persistence_layer = DynamoDBPersistenceLayer( When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: -Parameter | Required | Default | Description -------------------------------------------------- | ------------------------------------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------- -**table_name** | :heavy_check_mark: | | Table name to store state -**key_attr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) -**expiry_attr** | | `expiration` | Unix timestamp of when record expires -**in_progress_expiry_attr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) -**status_attr** | | `status` | Stores status of the lambda execution during and after invocation -**data_attr** | | `data` | Stores results of successfully executed Lambda handlers -**validation_key_attr** | | `validation` | Hashed representation of the parts of the event used for validation -**sort_key_attr** | | | Sort key of the table (if table is configured with a sort key). -**static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. +| Parameter | Required | Default | Description | +| --------------------------- | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +| **table_name** | :heavy_check_mark: | | Table name to store state | +| **key_attr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | +| **expiry_attr** | | `expiration` | Unix timestamp of when record expires | +| **in_progress_expiry_attr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) | +| **status_attr** | | `status` | Stores status of the lambda execution during and after invocation | +| **data_attr** | | `data` | Stores results of successfully executed Lambda handlers | +| **validation_key_attr** | | `validation` | Hashed representation of the parts of the event used for validation | +| **sort_key_attr** | | | Sort key of the table (if table is configured with a sort key). | +| **static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | ## Advanced @@ -554,15 +556,15 @@ Parameter | Required | Default | Description Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous example. These are the available options for further configuration -Parameter | Default | Description -------------------------------------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------- -**event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions) -**payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload -**raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request -**expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired -**use_local_cache** | `False` | Whether to locally cache idempotency results -**local_cache_max_items** | 256 | Max number of items to store in local cache -**hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library. +| Parameter | Default | Description | +| ------------------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions) | +| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | +| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | +| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | +| **use_local_cache** | `False` | Whether to locally cache idempotency results | +| **local_cache_max_items** | 256 | Max number of items to store in local cache | +| **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library. | ### Handling concurrent executions with the same payload @@ -826,11 +828,11 @@ def handler(event, context): The example function above would cause data to be stored in DynamoDB like this: -| id | sort_key | expiration | status | data | -|------------------------------|----------------------------------|------------|-------------|-------------------------------------| -| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"id": 12391, "message": "success"} | -| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"id": 527212, "message": "success"}| -| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | +| id | sort_key | expiration | status | data | +| ---------------------------- | -------------------------------- | ---------- | ----------- | ------------------------------------ | +| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"id": 12391, "message": "success"} | +| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"id": 527212, "message": "success"} | +| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | ### Bring your own persistent store From 84280af026ca1016e22e4f67fd7bdec0ffd72c6e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 29 Jul 2022 15:47:43 +0200 Subject: [PATCH 26/26] docs(idempotency): shorten wording to fit new mermaid SVG --- docs/utilities/idempotency.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 9823d1805a6..7ba61fd3062 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -364,17 +364,17 @@ sequenceDiagram participant Client participant Lambda participant Persistence Layer - alt initial request: + alt initial request Client->>Lambda: Invoke (event) Lambda->>Persistence Layer: Get or set (id=event.search(payload)) activate Persistence Layer - Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. - Lambda-->>Lambda: Run Lambda handler (event) - Lambda->>Persistence Layer: Update record with Lambda handler results + Note right of Persistence Layer: Locked to prevent concurrent
invocations with
the same payload. + Lambda-->>Lambda: Call handler (event) + Lambda->>Persistence Layer: Update record with result deactivate Persistence Layer Persistence Layer-->>Persistence Layer: Update record with result Lambda-->>Client: Response sent to client - else retried request: + else retried request Client->>Lambda: Invoke (event) Lambda->>Persistence Layer: Get or set (id=event.search(payload)) Persistence Layer-->>Lambda: Already exists in persistence layer. Return result @@ -437,26 +437,27 @@ sequenceDiagram participant Client participant Lambda participant Persistence Layer - alt initial request: + alt initial request Client->>Lambda: Invoke (event) Lambda->>Persistence Layer: Get or set (id=event.search(payload)) activate Persistence Layer - Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. - Lambda--xLambda: Run Lambda handler (event).
Time out + Note right of Persistence Layer: Locked to prevent concurrent
invocations with
the same payload. + Note over Lambda: Time out + Lambda--xLambda: Call handler (event) Lambda-->>Client: Return error response deactivate Persistence Layer - else retried before the Lambda timeout: + else concurrent request before timeout Client->>Lambda: Invoke (event) Lambda->>Persistence Layer: Get or set (id=event.search(payload)) - Persistence Layer-->>Lambda: Already exists in persistence layer. Still in-progress - Lambda--xClient: Return Invocation Already In Progress error - else retried after the Lambda timeout: + Persistence Layer-->>Lambda: Request already INPROGRESS + Lambda--xClient: Return IdempotencyAlreadyInProgressError + else retry after Lambda timeout Client->>Lambda: Invoke (event) Lambda->>Persistence Layer: Get or set (id=event.search(payload)) activate Persistence Layer - Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. - Lambda-->>Lambda: Run Lambda handler (event) - Lambda->>Persistence Layer: Update record with Lambda handler results + Note right of Persistence Layer: Locked to prevent concurrent
invocations with
the same payload. + Lambda-->>Lambda: Call handler (event) + Lambda->>Persistence Layer: Update record with result deactivate Persistence Layer Persistence Layer-->>Persistence Layer: Update record with result Lambda-->>Client: Response sent to client @@ -480,7 +481,7 @@ sequenceDiagram Lambda->>Persistence Layer: Get or set (id=event.search(payload)) activate Persistence Layer Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. - Lambda--xLambda: Run Lambda handler (event).
Raises exception + Lambda--xLambda: Call handler (event).
Raises exception Lambda->>Persistence Layer: Delete record (id=event.search(payload)) deactivate Persistence Layer Lambda-->>Client: Return error response