From 8d5808cf4facb3b45f5413f5f79ce3677805ef35 Mon Sep 17 00:00:00 2001 From: Nicholas Hakmiller Date: Thu, 6 Apr 2023 10:27:09 -0700 Subject: [PATCH 1/4] added force ttl check functionality to kv-table functions --- global_helpers/panther_oss_helpers.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/global_helpers/panther_oss_helpers.py b/global_helpers/panther_oss_helpers.py index a695f274a..004164b9e 100644 --- a/global_helpers/panther_oss_helpers.py +++ b/global_helpers/panther_oss_helpers.py @@ -3,7 +3,7 @@ import os import re import time -from datetime import datetime +from datetime import datetime, timedelta from ipaddress import ip_address from typing import Any, Dict, Optional, Sequence, Set, Union @@ -185,13 +185,22 @@ def kv_table() -> boto3.resource: ).Table("panther-kv-store") return _KV_TABLE +def ttl_expired(response: dict) -> bool: + """Checks whether a response from our panther-kv table has passed it's TTL date""" + """This can be used when the TTL is very exacting and trusting to DDB's cleanup cycle is insufficient""" + expiration = response.get("Item", {}).get("expiresAt", 0) + if expiration and float(expiration) <= (datetime.now()).timestamp(): + return True + return False -def get_counter(key: str) -> int: +def get_counter(key: str, force_ttl_check: bool = False) -> int: """Get a counter's current value (defaulting to 0 if key does not exist).""" response = kv_table().get_item( Key={"key": key}, ProjectionExpression=_COUNT_COL, ) + if force_ttl_check and ttl_expired(response): + return 0 return response.get("Item", {}).get(_COUNT_COL, 0) @@ -271,7 +280,7 @@ def put_dictionary(key: str, val: dict, epoch_seconds: int = None): set_key_expiration(key, epoch_seconds) -def get_dictionary(key: str) -> dict: +def get_dictionary(key: str, force_ttl_check: bool = False) -> dict: # Retrieve the item from DynamoDB response = kv_table().get_item(Key={"key": key}) @@ -281,6 +290,9 @@ def get_dictionary(key: str) -> dict: if not item: return {} + if force_ttl_check and ttl_expired(response): + return {} + try: # Deserialize from JSON to a Python dictionary return json.loads(item) @@ -291,12 +303,14 @@ def get_dictionary(key: str) -> dict: ) from exc -def get_string_set(key: str) -> Set[str]: +def get_string_set(key: str, force_ttl_check: bool = False) -> Set[str]: """Get a string set's current value (defaulting to empty set if key does not exit).""" response = kv_table().get_item( Key={"key": key}, ProjectionExpression=_STRING_SET_COL, ) + if force_ttl_check and ttl_expired(response): + return set() return response.get("Item", {}).get(_STRING_SET_COL, set()) From 29a8541691667b9f7255ba71a3fd9d3fe12362b8 Mon Sep 17 00:00:00 2001 From: Nicholas Hakmiller Date: Thu, 6 Apr 2023 11:26:50 -0700 Subject: [PATCH 2/4] linting & formatting --- global_helpers/panther_oss_helpers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/global_helpers/panther_oss_helpers.py b/global_helpers/panther_oss_helpers.py index 004164b9e..dcfadeec7 100644 --- a/global_helpers/panther_oss_helpers.py +++ b/global_helpers/panther_oss_helpers.py @@ -3,7 +3,7 @@ import os import re import time -from datetime import datetime, timedelta +from datetime import datetime from ipaddress import ip_address from typing import Any, Dict, Optional, Sequence, Set, Union @@ -185,14 +185,16 @@ def kv_table() -> boto3.resource: ).Table("panther-kv-store") return _KV_TABLE + def ttl_expired(response: dict) -> bool: - """Checks whether a response from our panther-kv table has passed it's TTL date""" - """This can be used when the TTL is very exacting and trusting to DDB's cleanup cycle is insufficient""" + """Checks whether a response from the panther-kv table has passed it's TTL date""" + # This can be used when the TTL timing is very exacting and DDB's cleanup is too slow expiration = response.get("Item", {}).get("expiresAt", 0) if expiration and float(expiration) <= (datetime.now()).timestamp(): return True return False + def get_counter(key: str, force_ttl_check: bool = False) -> int: """Get a counter's current value (defaulting to 0 if key does not exist).""" response = kv_table().get_item( From 6f4bc556610eda631690ff19beebf2fc3f7a877a Mon Sep 17 00:00:00 2001 From: Nicholas Hakmiller Date: Thu, 6 Apr 2023 11:28:06 -0700 Subject: [PATCH 3/4] pr comment --- global_helpers/panther_oss_helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/global_helpers/panther_oss_helpers.py b/global_helpers/panther_oss_helpers.py index dcfadeec7..557e41a5a 100644 --- a/global_helpers/panther_oss_helpers.py +++ b/global_helpers/panther_oss_helpers.py @@ -190,9 +190,7 @@ def ttl_expired(response: dict) -> bool: """Checks whether a response from the panther-kv table has passed it's TTL date""" # This can be used when the TTL timing is very exacting and DDB's cleanup is too slow expiration = response.get("Item", {}).get("expiresAt", 0) - if expiration and float(expiration) <= (datetime.now()).timestamp(): - return True - return False + return expiration and float(expiration) <= (datetime.now()).timestamp() def get_counter(key: str, force_ttl_check: bool = False) -> int: From 72a9b42e8137426b4ebf2536206d9c545954d6a8 Mon Sep 17 00:00:00 2001 From: Nicholas Hakmiller Date: Thu, 6 Apr 2023 11:59:59 -0700 Subject: [PATCH 4/4] add ttl column to get requests where missing --- global_helpers/panther_oss_helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/global_helpers/panther_oss_helpers.py b/global_helpers/panther_oss_helpers.py index 557e41a5a..48f64f1db 100644 --- a/global_helpers/panther_oss_helpers.py +++ b/global_helpers/panther_oss_helpers.py @@ -171,6 +171,7 @@ def resource_lookup(resource_id: str) -> Dict[str, Any]: _COUNT_COL = "intCount" _STRING_SET_COL = "stringSet" _DICT_COL = "dictionary" +_TTL_COL = "expiresAt" def kv_table() -> boto3.resource: @@ -189,7 +190,7 @@ def kv_table() -> boto3.resource: def ttl_expired(response: dict) -> bool: """Checks whether a response from the panther-kv table has passed it's TTL date""" # This can be used when the TTL timing is very exacting and DDB's cleanup is too slow - expiration = response.get("Item", {}).get("expiresAt", 0) + expiration = response.get("Item", {}).get(_TTL_COL, 0) return expiration and float(expiration) <= (datetime.now()).timestamp() @@ -197,7 +198,7 @@ def get_counter(key: str, force_ttl_check: bool = False) -> int: """Get a counter's current value (defaulting to 0 if key does not exist).""" response = kv_table().get_item( Key={"key": key}, - ProjectionExpression=_COUNT_COL, + ProjectionExpression=f"{_COUNT_COL}, {_TTL_COL}", ) if force_ttl_check and ttl_expired(response): return 0 @@ -307,7 +308,7 @@ def get_string_set(key: str, force_ttl_check: bool = False) -> Set[str]: """Get a string set's current value (defaulting to empty set if key does not exit).""" response = kv_table().get_item( Key={"key": key}, - ProjectionExpression=_STRING_SET_COL, + ProjectionExpression=f"{_STRING_SET_COL}, {_TTL_COL}", ) if force_ttl_check and ttl_expired(response): return set()