From 09855f9494ec113ed0c45ce154cda6af119ad923 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 15 Mar 2023 21:54:12 +0100 Subject: [PATCH 01/12] Initial sketch of DDB params provider --- powertools-parameters/pom.xml | 4 ++ .../parameters/DynamoDBProvider.java | 69 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java diff --git a/powertools-parameters/pom.xml b/powertools-parameters/pom.xml index 66eedce1e..befb57d09 100644 --- a/powertools-parameters/pom.xml +++ b/powertools-parameters/pom.xml @@ -73,6 +73,10 @@ software.amazon.awssdk url-connection-client + + software.amazon.awssdk + dynamodb + com.fasterxml.jackson.core jackson-databind diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java new file mode 100644 index 000000000..8cc46604d --- /dev/null +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java @@ -0,0 +1,69 @@ +package software.amazon.lambda.powertools.parameters; + +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; + +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; + +public class DynamoDBProvider extends BaseProvider { + + private final DynamoDbClient client; + private final String tableName; + + public DynamoDBProvider(CacheManager cacheManager, String tableName) { + this(cacheManager, DynamoDbClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) + .build(), + tableName + ); + + } + + DynamoDBProvider(CacheManager cacheManager, DynamoDbClient client, String tableName) { + super(cacheManager); + this.client = client; + this.tableName = tableName; + } + + @Override + protected String getValue(String key) { + GetItemResponse resp = client.getItem(GetItemRequest.builder() + .tableName(tableName) + .key(Collections.singletonMap("id", AttributeValue.fromS(key))) + .attributesToGet("val") + .build()); + + // TODO + return resp.item().values().stream().findFirst().get().s(); + } + + @Override + protected Map getMultipleValues(String path) { + + QueryResponse resp = client.query(QueryRequest.builder() + .tableName(tableName) + .keyConditionExpression("id = :v_id") + .expressionAttributeValues(Collections.singletonMap("v_id", AttributeValue.fromS(path))) + .attributesToGet("sk", "value") + .build()); + + return resp + .items() + .stream() + .collect( + Collectors.toMap( + (i) -> i.get("sk").s(), + (i) -> i.get("val").s())); + + + } +} From 8611acd5d57e79474a6fc72f144cb5edb1a4f6a6 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 16 Mar 2023 20:21:15 +0100 Subject: [PATCH 02/12] Add some E2E testing --- .../parameters/DynamoDBProvider.java | 17 ++-- .../parameters/DynamoDBProviderE2ETest.java | 89 +++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java index 8cc46604d..e33723e93 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java @@ -20,8 +20,8 @@ public class DynamoDBProvider extends BaseProvider { public DynamoDBProvider(CacheManager cacheManager, String tableName) { this(cacheManager, DynamoDbClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) - .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) - .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) + //.credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + //.region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) .build(), tableName ); @@ -42,8 +42,14 @@ protected String getValue(String key) { .attributesToGet("val") .build()); - // TODO - return resp.item().values().stream().findFirst().get().s(); + // If we have an item at the key, we should be able to get a 'val' out of it. If not it's + // exceptional. + // If we don't have an item at the key, we should return null. + if (resp.hasItem() && !resp.item().values().isEmpty()) { + return resp.item().values().stream().findFirst().get().s(); + } + + return null; } @Override @@ -52,8 +58,7 @@ protected Map getMultipleValues(String path) { QueryResponse resp = client.query(QueryRequest.builder() .tableName(tableName) .keyConditionExpression("id = :v_id") - .expressionAttributeValues(Collections.singletonMap("v_id", AttributeValue.fromS(path))) - .attributesToGet("sk", "value") + .expressionAttributeValues(Collections.singletonMap(":v_id", AttributeValue.fromS(path))) .build()); return resp diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java new file mode 100644 index 000000000..078d394a9 --- /dev/null +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java @@ -0,0 +1,89 @@ +package software.amazon.lambda.powertools.parameters; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * This class provides simple end-to-end style testing of the DynamoDBProvider class. + * It is ignored, for now, as it requires AWS access and that's not yet run as part + * of our unit test suite in the cloud. + * + * The test is kept here for 1/ local development and 2/ in preparation for future + * E2E tests running in the cloud CI. + */ +@Disabled +public class DynamoDBProviderE2ETest { + + final String ParamsTestTable = "ddb-params-test"; + final String MultiparamsTestTable = "ddb-multiparams-test"; + private DynamoDbClient ddbClient; + + public DynamoDBProviderE2ETest() { + // Create a DDB client to inject test data into our test tables + ddbClient = DynamoDbClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .build(); + } + + @Test + public void TestGetValue() { + + // Arrange + HashMap testItem = new HashMap(); + testItem.put("id", AttributeValue.fromS("test_param")); + testItem.put("val", AttributeValue.fromS("the_value_is_hello!")); + ddbClient.putItem(PutItemRequest.builder() + .tableName(ParamsTestTable) + .item(testItem) + .build()); + + // Act + DynamoDBProvider provider = new DynamoDBProvider(new CacheManager(), ParamsTestTable); + + // Assert + String value = provider.getValue("test_param"); + assertThat(value).isEqualTo("the_value_is_hello!"); + } + + @Test + public void TestGetValues() { + + // Arrange + HashMap testItem = new HashMap(); + testItem.put("id", AttributeValue.fromS("test_param")); + testItem.put("sk", AttributeValue.fromS("test_param_part_1")); + testItem.put("val", AttributeValue.fromS("the_value_is_hello!")); + ddbClient.putItem(PutItemRequest.builder() + .tableName(MultiparamsTestTable) + .item(testItem) + .build()); + + HashMap testItem2 = new HashMap(); + testItem2.put("id", AttributeValue.fromS("test_param")); + testItem2.put("sk", AttributeValue.fromS("test_param_part_2")); + testItem2.put("val", AttributeValue.fromS("the_value_is_still_hello!")); + ddbClient.putItem(PutItemRequest.builder() + .tableName(MultiparamsTestTable) + .item(testItem2) + .build()); + + // Act + DynamoDBProvider provider = new DynamoDBProvider(new CacheManager(), MultiparamsTestTable); + Map values = provider.getMultipleValues("test_param"); + + // Assert + assertThat(values.size()).isEqualTo(2); + assertThat(values.get("test_param_part_1")).isEqualTo("the_value_is_hello!"); + assertThat(values.get("test_param_part_2")).isEqualTo("the_value_is_still_hello!"); + } +} From 86fca8e15ffd932ce96170c143dab0d76278ce5f Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 16 Mar 2023 20:53:58 +0100 Subject: [PATCH 03/12] More tests --- .../parameters/DynamoDBProvider.java | 2 +- .../parameters/DynamoDBProviderTest.java | 131 ++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java index e33723e93..639061e79 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java @@ -46,7 +46,7 @@ protected String getValue(String key) { // exceptional. // If we don't have an item at the key, we should return null. if (resp.hasItem() && !resp.item().values().isEmpty()) { - return resp.item().values().stream().findFirst().get().s(); + return resp.item().get("val").s(); } return null; diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java new file mode 100644 index 000000000..0da28fb40 --- /dev/null +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java @@ -0,0 +1,131 @@ +package software.amazon.lambda.powertools.parameters; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.MockitoAnnotations.openMocks; + +public class DynamoDBProviderTest { + + @Mock + DynamoDbClient client; + + @Captor + ArgumentCaptor getItemValueCaptor; + + @Captor + ArgumentCaptor queryRequestCaptor; + + + private DynamoDBProvider provider; + private final String tableName = "ddb-test-table"; + + @BeforeEach + public void init() { + openMocks(this); + CacheManager cacheManager = new CacheManager(); + provider = new DynamoDBProvider(cacheManager, client, tableName); + } + + + @Test + public void getValue() { + + // Arrange + String key = "Key1"; + String expectedValue = "Value1"; + HashMap responseData = new HashMap(); + responseData.put("id", AttributeValue.fromS(key)); + responseData.put("val", AttributeValue.fromS(expectedValue)); + GetItemResponse response = GetItemResponse.builder() + .item(responseData) + .build(); + Mockito.when(client.getItem(getItemValueCaptor.capture())).thenReturn(response); + + // Act + String value = provider.getValue(key); + + // Assert + assertThat(value).isEqualTo(expectedValue); + assertThat(getItemValueCaptor.getValue().tableName()).isEqualTo(tableName); + assertThat(getItemValueCaptor.getValue().key().get("id").s()).isEqualTo(key); + } + + + @Test + public void getValueWithoutResultsReturnsNull() { + // Arrange + Mockito.when(client.getItem(getItemValueCaptor.capture())).thenReturn(GetItemResponse.builder() + .item(null) + .build()); + + // Act + String value = provider.getValue("key"); + + // Assert + assertThat(value).isEqualTo(null); + } + + @Test + public void getValueWithMalformedRowThrows() { + // Arrange + String key = "Key1"; + HashMap responseData = new HashMap(); + responseData.put("id", AttributeValue.fromS(key)); + responseData.put("not-val", AttributeValue.fromS("something")); + Mockito.when(client.getItem(getItemValueCaptor.capture())).thenReturn(GetItemResponse.builder() + .item(responseData) + .build()); + // Act + Assertions.assertThrows(NullPointerException.class, () -> { + String value = provider.getValue(key); + }); + } + + + @Test + public void getValues() { + + // Arrange + String key = "Key1"; + String subkey1 = "Subkey1"; + String val1 = "Val1"; + String subkey2 = "Subkey2"; + String val2 = "Val2"; + HashMap item1 = new HashMap(); + item1.put("id", AttributeValue.fromS(key)); + item1.put("sk", AttributeValue.fromS(subkey1)); + item1.put("val", AttributeValue.fromS(val1)); + HashMap item2 = new HashMap(); + item2.put("id", AttributeValue.fromS(key)); + item2.put("sk", AttributeValue.fromS(subkey2)); + item2.put("val", AttributeValue.fromS(val2)); + QueryResponse response = QueryResponse.builder() + .items(item1, item2) + .build(); + Mockito.when(client.query(queryRequestCaptor.capture())).thenReturn(response); + + // Act + Map values = provider.getMultipleValues(key); + + // Assert + assertThat(values.size()).isEqualTo(2); + assertThat(values.get(subkey1)).isEqualTo(val1); + assertThat(values.get(subkey2)).isEqualTo(val2); + assertThat(queryRequestCaptor.getValue().tableName()).isEqualTo(tableName); + assertThat(queryRequestCaptor.getValue().keyConditionExpression()).isEqualTo("id = :v_id"); + assertThat(queryRequestCaptor.getValue().expressionAttributeValues().get(":v_id").s()).isEqualTo(key); + } +} From c8793dad402e00992cc193d402f9bae954bcb940 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 08:49:58 +0100 Subject: [PATCH 04/12] More testing --- .../parameters/DynamoDBProvider.java | 23 ++++++++++- .../parameters/DynamoDBProviderE2ETest.java | 17 +++++++-- .../parameters/DynamoDBProviderTest.java | 38 ++++++++++++++++++- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java index 639061e79..07254848f 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java @@ -12,6 +12,13 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * Implements a {@link ParamProvider} on top of DynamoDB. The schema of the table + * is described in the Python powertools documentation. + * + * @see Python DynamoDB provider + * + */ public class DynamoDBProvider extends BaseProvider { private final DynamoDbClient client; @@ -20,8 +27,8 @@ public class DynamoDBProvider extends BaseProvider { public DynamoDBProvider(CacheManager cacheManager, String tableName) { this(cacheManager, DynamoDbClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) - //.credentialsProvider(EnvironmentVariableCredentialsProvider.create()) - //.region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) .build(), tableName ); @@ -34,6 +41,12 @@ public DynamoDBProvider(CacheManager cacheManager, String tableName) { this.tableName = tableName; } + /** + * Return a single value from the DynamoDB parameter provider. + * + * @param key key of the parameter + * @return The value, if it exists, null if it doesn't. Throws if the row exists but doesn't match the schema. + */ @Override protected String getValue(String key) { GetItemResponse resp = client.getItem(GetItemRequest.builder() @@ -52,6 +65,12 @@ protected String getValue(String key) { return null; } + /** + * Returns multiple values from the DynamoDB parameter provider. + * + * @param path Parameter store path + * @return All values matching the given path, and an empty map if none do. Throws if any records exist that don't match the schema. + */ @Override protected Map getMultipleValues(String path) { diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java index 078d394a9..601f56925 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java @@ -26,13 +26,15 @@ public class DynamoDBProviderE2ETest { final String ParamsTestTable = "ddb-params-test"; final String MultiparamsTestTable = "ddb-multiparams-test"; - private DynamoDbClient ddbClient; + private final DynamoDbClient ddbClient; public DynamoDBProviderE2ETest() { // Create a DDB client to inject test data into our test tables ddbClient = DynamoDbClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) .build(); + + } @Test @@ -48,10 +50,10 @@ public void TestGetValue() { .build()); // Act - DynamoDBProvider provider = new DynamoDBProvider(new CacheManager(), ParamsTestTable); + DynamoDBProvider provider = makeProvider(ParamsTestTable); + String value = provider.getValue("test_param"); // Assert - String value = provider.getValue("test_param"); assertThat(value).isEqualTo("the_value_is_hello!"); } @@ -78,7 +80,7 @@ public void TestGetValues() { .build()); // Act - DynamoDBProvider provider = new DynamoDBProvider(new CacheManager(), MultiparamsTestTable); + DynamoDBProvider provider = makeProvider(MultiparamsTestTable); Map values = provider.getMultipleValues("test_param"); // Assert @@ -86,4 +88,11 @@ public void TestGetValues() { assertThat(values.get("test_param_part_1")).isEqualTo("the_value_is_hello!"); assertThat(values.get("test_param_part_2")).isEqualTo("the_value_is_still_hello!"); } + + private DynamoDBProvider makeProvider(String tableName) { + return new DynamoDBProvider(new CacheManager(), DynamoDbClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()).build(), + tableName); + } + } diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java index 0da28fb40..1253362ea 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java @@ -13,6 +13,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.MockitoAnnotations.openMocks; @@ -95,7 +96,7 @@ public void getValueWithMalformedRowThrows() { } - @Test + @Test public void getValues() { // Arrange @@ -128,4 +129,39 @@ public void getValues() { assertThat(queryRequestCaptor.getValue().keyConditionExpression()).isEqualTo("id = :v_id"); assertThat(queryRequestCaptor.getValue().expressionAttributeValues().get(":v_id").s()).isEqualTo(key); } + + @Test + public void getValuesWithoutResultsReturnsNull() { + // Arrange + Mockito.when(client.query(queryRequestCaptor.capture())).thenReturn( + QueryResponse.builder().items().build()); + + // Act + Map values = provider.getMultipleValues(UUID.randomUUID().toString()); + + // Assert + assertThat(values.size()).isEqualTo(0); + } + + @Test + public void getValuesWithMalformedRowThrows() { + // Arrange + String key = "Key1"; + HashMap item1 = new HashMap(); + item1.put("id", AttributeValue.fromS(key)); + item1.put("sk", AttributeValue.fromS("some-subkey")); + item1.put("notVal", AttributeValue.fromS("somevalue")); + QueryResponse response = QueryResponse.builder() + .items(item1) + .build(); + Mockito.when(client.query(queryRequestCaptor.capture())).thenReturn(response); + + // Assert + Assertions.assertThrows(NullPointerException.class, () -> { + // Act + Map values = provider.getMultipleValues(key); + }); + } + + } From 546d77dbd854876024b347d74459fef87f99a06b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 10:11:52 +0100 Subject: [PATCH 05/12] Plumbing and renaming bits --- .../parameters/DynamoDBProvider.java | 93 --------- .../parameters/DynamoDbProvider.java | 179 ++++++++++++++++++ .../powertools/parameters/ParamManager.java | 24 ++- ...Test.java => DynamoDbProviderE2ETest.java} | 12 +- ...derTest.java => DynamoDbProviderTest.java} | 6 +- 5 files changed, 210 insertions(+), 104 deletions(-) delete mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java create mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java rename powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/{DynamoDBProviderE2ETest.java => DynamoDbProviderE2ETest.java} (91%) rename powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/{DynamoDBProviderTest.java => DynamoDbProviderTest.java} (97%) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java deleted file mode 100644 index 07254848f..000000000 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDBProvider.java +++ /dev/null @@ -1,93 +0,0 @@ -package software.amazon.lambda.powertools.parameters; - -import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; -import software.amazon.awssdk.core.SdkSystemSetting; -import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.*; -import software.amazon.lambda.powertools.parameters.cache.CacheManager; - -import java.util.Collections; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Implements a {@link ParamProvider} on top of DynamoDB. The schema of the table - * is described in the Python powertools documentation. - * - * @see Python DynamoDB provider - * - */ -public class DynamoDBProvider extends BaseProvider { - - private final DynamoDbClient client; - private final String tableName; - - public DynamoDBProvider(CacheManager cacheManager, String tableName) { - this(cacheManager, DynamoDbClient.builder() - .httpClientBuilder(UrlConnectionHttpClient.builder()) - .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) - .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) - .build(), - tableName - ); - - } - - DynamoDBProvider(CacheManager cacheManager, DynamoDbClient client, String tableName) { - super(cacheManager); - this.client = client; - this.tableName = tableName; - } - - /** - * Return a single value from the DynamoDB parameter provider. - * - * @param key key of the parameter - * @return The value, if it exists, null if it doesn't. Throws if the row exists but doesn't match the schema. - */ - @Override - protected String getValue(String key) { - GetItemResponse resp = client.getItem(GetItemRequest.builder() - .tableName(tableName) - .key(Collections.singletonMap("id", AttributeValue.fromS(key))) - .attributesToGet("val") - .build()); - - // If we have an item at the key, we should be able to get a 'val' out of it. If not it's - // exceptional. - // If we don't have an item at the key, we should return null. - if (resp.hasItem() && !resp.item().values().isEmpty()) { - return resp.item().get("val").s(); - } - - return null; - } - - /** - * Returns multiple values from the DynamoDB parameter provider. - * - * @param path Parameter store path - * @return All values matching the given path, and an empty map if none do. Throws if any records exist that don't match the schema. - */ - @Override - protected Map getMultipleValues(String path) { - - QueryResponse resp = client.query(QueryRequest.builder() - .tableName(tableName) - .keyConditionExpression("id = :v_id") - .expressionAttributeValues(Collections.singletonMap(":v_id", AttributeValue.fromS(path))) - .build()); - - return resp - .items() - .stream() - .collect( - Collectors.toMap( - (i) -> i.get("sk").s(), - (i) -> i.get("val").s())); - - - } -} diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java new file mode 100644 index 000000000..30b2f3049 --- /dev/null +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java @@ -0,0 +1,179 @@ +package software.amazon.lambda.powertools.parameters; + +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import software.amazon.lambda.powertools.parameters.transform.TransformationManager; + +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Implements a {@link ParamProvider} on top of DynamoDB. The schema of the table + * is described in the Python powertools documentation. + * + * @see Python DynamoDB provider + * + */ +public class DynamoDbProvider extends BaseProvider { + + private final DynamoDbClient client; + private final String tableName; + + public DynamoDbProvider(CacheManager cacheManager, String tableName) { + this(cacheManager, DynamoDbClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) + .build(), + tableName + ); + + } + + DynamoDbProvider(CacheManager cacheManager, DynamoDbClient client, String tableName) { + super(cacheManager); + this.client = client; + this.tableName = tableName; + } + + /** + * Return a single value from the DynamoDB parameter provider. + * + * @param key key of the parameter + * @return The value, if it exists, null if it doesn't. Throws if the row exists but doesn't match the schema. + */ + @Override + protected String getValue(String key) { + GetItemResponse resp = client.getItem(GetItemRequest.builder() + .tableName(tableName) + .key(Collections.singletonMap("id", AttributeValue.fromS(key))) + .attributesToGet("val") + .build()); + + // If we have an item at the key, we should be able to get a 'val' out of it. If not it's + // exceptional. + // If we don't have an item at the key, we should return null. + if (resp.hasItem() && !resp.item().values().isEmpty()) { + return resp.item().get("val").s(); + } + + return null; + } + + /** + * Returns multiple values from the DynamoDB parameter provider. + * + * @param path Parameter store path + * @return All values matching the given path, and an empty map if none do. Throws if any records exist that don't match the schema. + */ + @Override + protected Map getMultipleValues(String path) { + + QueryResponse resp = client.query(QueryRequest.builder() + .tableName(tableName) + .keyConditionExpression("id = :v_id") + .expressionAttributeValues(Collections.singletonMap(":v_id", AttributeValue.fromS(path))) + .build()); + + return resp + .items() + .stream() + .collect( + Collectors.toMap( + (i) -> i.get("sk").s(), + (i) -> i.get("val").s())); + + + } + + /** + * Create a builder that can be used to configure and create a {@link DynamoDbProvider}. + * + * @return a new instance of {@link DynamoDbProvider.Builder} + */ + public static DynamoDbProvider.Builder builder() { + return new DynamoDbProvider.Builder(); + } + + static class Builder { + private DynamoDbClient client; + private String table; + private CacheManager cacheManager; + private TransformationManager transformationManager; + + /** + * Create a {@link DynamoDbProvider} instance. + * + * @return a {@link DynamoDbProvider} + */ + public DynamoDbProvider build() { + if (cacheManager == null) { + throw new IllegalStateException("No CacheManager provided; please provide one"); + } + if (table == null) { + throw new IllegalStateException("No DynamoDB table name provided; please provide one"); + } + DynamoDbProvider provider; + if (client != null) { + provider = new DynamoDbProvider(cacheManager, client, table); + } else { + provider = new DynamoDbProvider(cacheManager, table); + } + if (transformationManager != null) { + provider.setTransformationManager(transformationManager); + } + return provider; + } + + /** + * Set custom {@link DynamoDbClient} to pass to the {@link DynamoDbClient}.
+ * Use it if you want to customize the region or any other part of the client. + * + * @param client Custom client + * @return the builder to chain calls (eg. 
builder.withClient().build()
) + */ + public DynamoDbProvider.Builder withClient(DynamoDbClient client) { + this.client = client; + return this; + } + + /** + * Mandatory. Provide a CacheManager to the {@link DynamoDbProvider} + * + * @param cacheManager the manager that will handle the cache of parameters + * @return the builder to chain calls (eg. 
builder.withCacheManager().build()
) + */ + public DynamoDbProvider.Builder withCacheManager(CacheManager cacheManager) { + this.cacheManager = cacheManager; + return this; + } + + /** + * Mandatory. Provide a DynamoDB table to the {@link DynamoDbProvider} + * + * @param table the table that parameters will be retrieved from. + * @return the builder to chain calls (eg. 
builder.withTable().build()
) + */ + public DynamoDbProvider.Builder withTable(String table) { + this.table = table; + return this; + } + + /** + * Provide a transformationManager to the {@link DynamoDbProvider} + * + * @param transformationManager the manager that will handle transformation of parameters + * @return the builder to chain calls (eg. 
builder.withTransformationManager().build()
) + */ + public DynamoDbProvider.Builder withTransformationManager(TransformationManager transformationManager) { + this.transformationManager = transformationManager; + return this; + } + } +} diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java index 0131ae179..c31d31b71 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java @@ -15,6 +15,7 @@ import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.ssm.SsmClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.lambda.powertools.parameters.cache.CacheManager; import software.amazon.lambda.powertools.parameters.transform.TransformationManager; @@ -36,8 +37,8 @@ public final class ParamManager { /** * Get a concrete implementation of {@link BaseProvider}.
- * You can specify {@link SecretsProvider} or {@link SSMProvider} or create your custom provider - * by extending {@link BaseProvider} if you need to integrate with a different parameter store. + * You can specify {@link SecretsProvider}, {@link SSMProvider}, {@link DynamoDbProvider}, or create your + * custom provider by extending {@link BaseProvider} if you need to integrate with a different parameter store. * @return a {@link SecretsProvider} */ public static T getProvider(Class providerClass) { @@ -65,6 +66,12 @@ public static SSMProvider getSsmProvider() { return getProvider(SSMProvider.class); } + /** + * Get a {@link DynamoDbProvider} with default {@link DynamoDbClient}
+ * If you need to customize the region, or other part of the client, use + */ + public static DynamoDbProvider getDynamodbProvider() { return getProvider(DynamoDbProvider.class); } + /** * Get a {@link SecretsProvider} with your custom {@link SecretsManagerClient}.
* Use this to configure region or other part of the client. Use {@link ParamManager#getSsmProvider()} if you don't need this customization. @@ -91,6 +98,19 @@ public static SSMProvider getSsmProvider(SsmClient client) { .build()); } + /** + * Get a {@link DynamoDbProvider} with your custom {@link DynamoDbClient}.
+ * Use this to configure region or other part of the client. Use {@link ParamManager#getDynamodbProvider()} if you don't need this customization. + * @return a {@link DynamoDbProvider} + */ + public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client) { + return (DynamoDbProvider) providers.computeIfAbsent(DynamoDbProvider.class, (k) -> DynamoDbProvider.builder() + .withClient(client) + .withCacheManager(cacheManager) + .withTransformationManager(transformationManager) + .build()); + } + public static CacheManager getCacheManager() { return cacheManager; } diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java similarity index 91% rename from powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java rename to powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java index 601f56925..0e427d1b0 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderE2ETest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java @@ -22,13 +22,13 @@ * E2E tests running in the cloud CI. */ @Disabled -public class DynamoDBProviderE2ETest { +public class DynamoDbProviderE2ETest { final String ParamsTestTable = "ddb-params-test"; final String MultiparamsTestTable = "ddb-multiparams-test"; private final DynamoDbClient ddbClient; - public DynamoDBProviderE2ETest() { + public DynamoDbProviderE2ETest() { // Create a DDB client to inject test data into our test tables ddbClient = DynamoDbClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) @@ -50,7 +50,7 @@ public void TestGetValue() { .build()); // Act - DynamoDBProvider provider = makeProvider(ParamsTestTable); + DynamoDbProvider provider = makeProvider(ParamsTestTable); String value = provider.getValue("test_param"); // Assert @@ -80,7 +80,7 @@ public void TestGetValues() { .build()); // Act - DynamoDBProvider provider = makeProvider(MultiparamsTestTable); + DynamoDbProvider provider = makeProvider(MultiparamsTestTable); Map values = provider.getMultipleValues("test_param"); // Assert @@ -89,8 +89,8 @@ public void TestGetValues() { assertThat(values.get("test_param_part_2")).isEqualTo("the_value_is_still_hello!"); } - private DynamoDBProvider makeProvider(String tableName) { - return new DynamoDBProvider(new CacheManager(), DynamoDbClient.builder() + private DynamoDbProvider makeProvider(String tableName) { + return new DynamoDbProvider(new CacheManager(), DynamoDbClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()).build(), tableName); } diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java similarity index 97% rename from powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java rename to powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java index 1253362ea..78c703c4f 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDBProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java @@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.MockitoAnnotations.openMocks; -public class DynamoDBProviderTest { +public class DynamoDbProviderTest { @Mock DynamoDbClient client; @@ -30,14 +30,14 @@ public class DynamoDBProviderTest { ArgumentCaptor queryRequestCaptor; - private DynamoDBProvider provider; + private DynamoDbProvider provider; private final String tableName = "ddb-test-table"; @BeforeEach public void init() { openMocks(this); CacheManager cacheManager = new CacheManager(); - provider = new DynamoDBProvider(cacheManager, client, tableName); + provider = new DynamoDbProvider(cacheManager, client, tableName); } From d9a7104ae5e836c04bb87a10bf98db660e7d8975 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 14:21:22 +0100 Subject: [PATCH 06/12] More tests and plumbing into ParamManager --- .../powertools/parameters/ParamManager.java | 18 ++++++++++++++---- .../ParamManagerIntegrationTest.java | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java index c31d31b71..0479391fe 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java @@ -68,9 +68,18 @@ public static SSMProvider getSsmProvider() { /** * Get a {@link DynamoDbProvider} with default {@link DynamoDbClient}
- * If you need to customize the region, or other part of the client, use + * If you need to customize the region, or other part of the client, use {@link ParamManager#getDynamoDbProvider(DynamoDbClient, String)} */ - public static DynamoDbProvider getDynamodbProvider() { return getProvider(DynamoDbProvider.class); } + public static DynamoDbProvider getDynamodbProvider(String tableName) { + // Because we need a DDB table name to configure our client, we can't use + // ParamManager#getProvider. This means that we need to make sure we do the same stuff - + // set transformation manager and cache manager. + return DynamoDbProvider.builder() + .withCacheManager(cacheManager) + .withTable(tableName) + .withTransformationManager(transformationManager) + .build(); + } /** * Get a {@link SecretsProvider} with your custom {@link SecretsManagerClient}.
@@ -100,12 +109,13 @@ public static SSMProvider getSsmProvider(SsmClient client) { /** * Get a {@link DynamoDbProvider} with your custom {@link DynamoDbClient}.
- * Use this to configure region or other part of the client. Use {@link ParamManager#getDynamodbProvider()} if you don't need this customization. + * Use this to configure region or other part of the client. Use {@link ParamManager#getDynamodbProvider(String)} )} if you don't need this customization. * @return a {@link DynamoDbProvider} */ - public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client) { + public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client, String table) { return (DynamoDbProvider) providers.computeIfAbsent(DynamoDbProvider.class, (k) -> DynamoDbProvider.builder() .withClient(client) + .withTable(table) .withCacheManager(cacheManager) .withTransformationManager(transformationManager) .build()); diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java index 0b4a2093f..ec1672ead 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java @@ -19,6 +19,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; @@ -41,6 +42,9 @@ public class ParamManagerIntegrationTest { @Mock SsmClient ssmClient; + @Mock + DynamoDbClient ddbClient; + @Captor ArgumentCaptor ssmParamCaptor; @@ -116,4 +120,16 @@ public void secretsProvider_get() { assertThat(secretsProvider.get("keys")).isEqualTo(expectedValue); // second time is from cache verify(secretsManagerClient, times(1)).getSecretValue(any(GetSecretValueRequest.class)); } + + @Test + public void getDynamoDbProvider() { + + // Act + DynamoDbProvider provider = ParamManager.getDynamoDbProvider(ddbClient, "test-table"); + + // Assert + assertThat(provider).isNotNull(); + + + } } From fced323c4128d30cfd7ee1ca58a74886c063035d Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 15:39:13 +0100 Subject: [PATCH 07/12] Add additional documentation --- docs/utilities/parameters.md | 39 +++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index e2d0cb965..7d3c93add 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -5,8 +5,9 @@ description: Utility The parameters utility provides a way to retrieve parameter values from -[AWS Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) or -[AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). It also provides a base class to create your parameter provider implementation. +[AWS Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html), +[AWS Secrets Manager](https://aws.amazon.com/secrets-manager/), or [Amazon DynamoDB](https://aws.amazon.com/dynamodb/). +It also provides a base class to create your parameter provider implementation. **Key features** @@ -40,11 +41,12 @@ To install this utility, add the following dependency to your project. This utility requires additional permissions to work as expected. See the table below: -Provider | Function/Method | IAM Permission -------------------------------------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------- -SSM Parameter Store | `SSMProvider.get(String)` `SSMProvider.get(String, Class)` | `ssm:GetParameter` -SSM Parameter Store | `SSMProvider.getMultiple(String)` | `ssm:GetParametersByPath` -Secrets Manager | `SecretsProvider.get(String)` `SecretsProvider.get(String, Class)` | `secretsmanager:GetSecretValue` +Provider | Function/Method | IAM Permission +------------------------------------------------- |----------------------------------------------------------------------| --------------------------------------------------------------------------------- +SSM Parameter Store | `SSMProvider.get(String)` `SSMProvider.get(String, Class)` | `ssm:GetParameter` +SSM Parameter Store | `SSMProvider.getMultiple(String)` | `ssm:GetParametersByPath` +Secrets Manager | `SecretsProvider.get(String)` `SecretsProvider.get(String, Class)` | `secretsmanager:GetSecretValue` +DynamoDB | `DynamoDBProvider.get(String)` `DynamoDBProvider.getMultiple(string)` | `dynamodb:GetItem` `dynamoDB:Query` ## SSM Parameter Store @@ -166,6 +168,29 @@ in order to get data from other regions or use specific credentials. } ``` +## DynamoDB +To get secrets stored in DynamoDB, use `getDynamoDbProvider`, providing the name of the table that +contains the secrets. As with the other providers, an overloaded methods allows you to retrieve +a `DynamoDbProvider` providing a client if you need to configure it yourself. + +=== "DynamoDbProvider" + + ```java hl_lines="9" + import software.amazon.lambda.powertools.parameters.DynamoDbProvider; + import software.amazon.lambda.powertools.parameters.ParamManager; + + public class AppWithDynamoDbParameters implements RequestHandler { + // Get an instance of the DynamoDbProvider + DynamoDbProvider ddbProvider = ParamManager.getDynamoDbProvider("my-parameters-table"); + + // Retrieve a single parameter + String value = ddbProvider.get("my-key"); + } + ``` + + + + ## Advanced configuration ### Caching From 97a7503a33067218cbd67d985608fd496b64838a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 15:42:41 +0100 Subject: [PATCH 08/12] More docs --- docs/utilities/parameters.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 7d3c93add..5cfffcfb0 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -175,7 +175,7 @@ a `DynamoDbProvider` providing a client if you need to configure it yourself. === "DynamoDbProvider" - ```java hl_lines="9" + ```java hl_lines="6 9" import software.amazon.lambda.powertools.parameters.DynamoDbProvider; import software.amazon.lambda.powertools.parameters.ParamManager; @@ -188,6 +188,27 @@ a `DynamoDbProvider` providing a client if you need to configure it yourself. } ``` +=== "DynamoDbProvider with an explicit region" + + ```java hl_lines="7-10 13 16" + import software.amazon.lambda.powertools.parameters.DynamoDbProvider; + import software.amazon.lambda.powertools.parameters.ParamManager; + import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + + public class AppWithDynamoDbParameters implements RequestHandler { + // Get a DynamoDB Client with an explicit region + DynamoDbClient ddbClient = DynamoDbClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .region(Region.EU_CENTRAL_2) + .build(); + + // Get an instance of the DynamoDbProvider + DynamoDbProvider provider = ParamManager.getDynamoDbProvider(ddbClient, "test-table"); + + // Retrieve a single parameter + String value = ddbProvider.get("my-key"); + } + ``` From aa5ab8e19e59ccea3b3a8469316066ba95c05af2 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 16:00:11 +0100 Subject: [PATCH 09/12] More docs --- docs/utilities/parameters.md | 4 +++- .../amazon/lambda/powertools/parameters/ParamManager.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 5cfffcfb0..aaad82622 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -190,10 +190,12 @@ a `DynamoDbProvider` providing a client if you need to configure it yourself. === "DynamoDbProvider with an explicit region" - ```java hl_lines="7-10 13 16" + ```java hl_lines="9 10 11 12 15 18" import software.amazon.lambda.powertools.parameters.DynamoDbProvider; import software.amazon.lambda.powertools.parameters.ParamManager; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; + import software.amazon.awssdk.regions.Region; public class AppWithDynamoDbParameters implements RequestHandler { // Get a DynamoDB Client with an explicit region diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java index 0479391fe..e4c70acb0 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java @@ -70,7 +70,7 @@ public static SSMProvider getSsmProvider() { * Get a {@link DynamoDbProvider} with default {@link DynamoDbClient}
* If you need to customize the region, or other part of the client, use {@link ParamManager#getDynamoDbProvider(DynamoDbClient, String)} */ - public static DynamoDbProvider getDynamodbProvider(String tableName) { + public static DynamoDbProvider getDynamoDbProvider(String tableName) { // Because we need a DDB table name to configure our client, we can't use // ParamManager#getProvider. This means that we need to make sure we do the same stuff - // set transformation manager and cache manager. @@ -109,7 +109,7 @@ public static SSMProvider getSsmProvider(SsmClient client) { /** * Get a {@link DynamoDbProvider} with your custom {@link DynamoDbClient}.
- * Use this to configure region or other part of the client. Use {@link ParamManager#getDynamodbProvider(String)} )} if you don't need this customization. + * Use this to configure region or other part of the client. Use {@link ParamManager#getDynamoDbProvider(String)} )} if you don't need this customization. * @return a {@link DynamoDbProvider} */ public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client, String table) { From 21386f5133bbcd58f8e4fae3505edaf82a93191e Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 17 Mar 2023 16:10:07 +0100 Subject: [PATCH 10/12] Fix doc link --- .../amazon/lambda/powertools/parameters/DynamoDbProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java index 30b2f3049..a1ba2ea57 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java @@ -15,9 +15,9 @@ /** * Implements a {@link ParamProvider} on top of DynamoDB. The schema of the table - * is described in the Python powertools documentation. + * is described in the Powertools documentation. * - * @see Python DynamoDB provider + * @see Parameters provider documentation * */ public class DynamoDbProvider extends BaseProvider { From 577129167b3255d4b5b5caa999fda0ee5f6b98fd Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 20 Mar 2023 08:32:25 +0100 Subject: [PATCH 11/12] Address jvdl review comments --- docs/utilities/parameters.md | 6 +++--- .../lambda/powertools/parameters/DynamoDbProvider.java | 6 +++--- .../powertools/parameters/DynamoDbProviderE2ETest.java | 9 +++++---- .../powertools/parameters/DynamoDbProviderTest.java | 10 +++++----- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index aaad82622..e8df1f8d6 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -76,7 +76,7 @@ in order to get data from other regions or use specific credentials. } ``` -=== "SSMProvider with an explicit region" +=== "SSMProvider with a custom client" ```java hl_lines="5 7" import software.amazon.lambda.powertools.parameters.SSMProvider; @@ -151,7 +151,7 @@ in order to get data from other regions or use specific credentials. } ``` -=== "SecretsProvider with an explicit region" +=== "SecretsProvider with a custom client" ```java hl_lines="5 7" import software.amazon.lambda.powertools.parameters.SecretsProvider; @@ -188,7 +188,7 @@ a `DynamoDbProvider` providing a client if you need to configure it yourself. } ``` -=== "DynamoDbProvider with an explicit region" +=== "DynamoDbProvider with a custom client" ```java hl_lines="9 10 11 12 15 18" import software.amazon.lambda.powertools.parameters.DynamoDbProvider; diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java index a1ba2ea57..91b284fe5 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java @@ -53,14 +53,14 @@ protected String getValue(String key) { GetItemResponse resp = client.getItem(GetItemRequest.builder() .tableName(tableName) .key(Collections.singletonMap("id", AttributeValue.fromS(key))) - .attributesToGet("val") + .attributesToGet("value") .build()); // If we have an item at the key, we should be able to get a 'val' out of it. If not it's // exceptional. // If we don't have an item at the key, we should return null. if (resp.hasItem() && !resp.item().values().isEmpty()) { - return resp.item().get("val").s(); + return resp.item().get("value").s(); } return null; @@ -87,7 +87,7 @@ protected Map getMultipleValues(String path) { .collect( Collectors.toMap( (i) -> i.get("sk").s(), - (i) -> i.get("val").s())); + (i) -> i.get("value").s())); } diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java index 0e427d1b0..c9397676b 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderE2ETest.java @@ -19,7 +19,8 @@ * of our unit test suite in the cloud. * * The test is kept here for 1/ local development and 2/ in preparation for future - * E2E tests running in the cloud CI. + * E2E tests running in the cloud CI. Once the E2E test structure is merged we + * will move this across. */ @Disabled public class DynamoDbProviderE2ETest { @@ -43,7 +44,7 @@ public void TestGetValue() { // Arrange HashMap testItem = new HashMap(); testItem.put("id", AttributeValue.fromS("test_param")); - testItem.put("val", AttributeValue.fromS("the_value_is_hello!")); + testItem.put("value", AttributeValue.fromS("the_value_is_hello!")); ddbClient.putItem(PutItemRequest.builder() .tableName(ParamsTestTable) .item(testItem) @@ -64,7 +65,7 @@ public void TestGetValues() { HashMap testItem = new HashMap(); testItem.put("id", AttributeValue.fromS("test_param")); testItem.put("sk", AttributeValue.fromS("test_param_part_1")); - testItem.put("val", AttributeValue.fromS("the_value_is_hello!")); + testItem.put("value", AttributeValue.fromS("the_value_is_hello!")); ddbClient.putItem(PutItemRequest.builder() .tableName(MultiparamsTestTable) .item(testItem) @@ -73,7 +74,7 @@ public void TestGetValues() { HashMap testItem2 = new HashMap(); testItem2.put("id", AttributeValue.fromS("test_param")); testItem2.put("sk", AttributeValue.fromS("test_param_part_2")); - testItem2.put("val", AttributeValue.fromS("the_value_is_still_hello!")); + testItem2.put("value", AttributeValue.fromS("the_value_is_still_hello!")); ddbClient.putItem(PutItemRequest.builder() .tableName(MultiparamsTestTable) .item(testItem2) diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java index 78c703c4f..aaee0038c 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java @@ -49,7 +49,7 @@ public void getValue() { String expectedValue = "Value1"; HashMap responseData = new HashMap(); responseData.put("id", AttributeValue.fromS(key)); - responseData.put("val", AttributeValue.fromS(expectedValue)); + responseData.put("value", AttributeValue.fromS(expectedValue)); GetItemResponse response = GetItemResponse.builder() .item(responseData) .build(); @@ -85,7 +85,7 @@ public void getValueWithMalformedRowThrows() { String key = "Key1"; HashMap responseData = new HashMap(); responseData.put("id", AttributeValue.fromS(key)); - responseData.put("not-val", AttributeValue.fromS("something")); + responseData.put("not-value", AttributeValue.fromS("something")); Mockito.when(client.getItem(getItemValueCaptor.capture())).thenReturn(GetItemResponse.builder() .item(responseData) .build()); @@ -108,11 +108,11 @@ public void getValues() { HashMap item1 = new HashMap(); item1.put("id", AttributeValue.fromS(key)); item1.put("sk", AttributeValue.fromS(subkey1)); - item1.put("val", AttributeValue.fromS(val1)); + item1.put("value", AttributeValue.fromS(val1)); HashMap item2 = new HashMap(); item2.put("id", AttributeValue.fromS(key)); item2.put("sk", AttributeValue.fromS(subkey2)); - item2.put("val", AttributeValue.fromS(val2)); + item2.put("value", AttributeValue.fromS(val2)); QueryResponse response = QueryResponse.builder() .items(item1, item2) .build(); @@ -150,7 +150,7 @@ public void getValuesWithMalformedRowThrows() { HashMap item1 = new HashMap(); item1.put("id", AttributeValue.fromS(key)); item1.put("sk", AttributeValue.fromS("some-subkey")); - item1.put("notVal", AttributeValue.fromS("somevalue")); + item1.put("not-value", AttributeValue.fromS("somevalue")); QueryResponse response = QueryResponse.builder() .items(item1) .build(); From 9eed4d0ee31d27ebb72be49fb311a2b1cc5e62b9 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 20 Mar 2023 08:43:47 +0100 Subject: [PATCH 12/12] Throw a more appropriate RuntimeException if DynamoDbProvider hits a schema issue --- .../powertools/parameters/DynamoDbProvider.java | 12 ++++++++++++ .../exception/DynamoDbProviderSchemaException.java | 11 +++++++++++ .../powertools/parameters/DynamoDbProviderTest.java | 5 +++-- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/DynamoDbProviderSchemaException.java diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java index 91b284fe5..08031e185 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/DynamoDbProvider.java @@ -7,6 +7,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.*; import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import software.amazon.lambda.powertools.parameters.exception.DynamoDbProviderSchemaException; import software.amazon.lambda.powertools.parameters.transform.TransformationManager; import java.util.Collections; @@ -60,6 +61,9 @@ protected String getValue(String key) { // exceptional. // If we don't have an item at the key, we should return null. if (resp.hasItem() && !resp.item().values().isEmpty()) { + if (!resp.item().containsKey("value")) { + throw new DynamoDbProviderSchemaException("Missing 'value': " + resp.item().toString()); + } return resp.item().get("value").s(); } @@ -84,6 +88,14 @@ protected Map getMultipleValues(String path) { return resp .items() .stream() + .peek((i) -> { + if (!i.containsKey("sk")) { + throw new DynamoDbProviderSchemaException("Missing 'sk': " + i.toString()); + } + if (!i.containsKey("value")) { + throw new DynamoDbProviderSchemaException("Missing 'value': " + i.toString()); + } + }) .collect( Collectors.toMap( (i) -> i.get("sk").s(), diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/DynamoDbProviderSchemaException.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/DynamoDbProviderSchemaException.java new file mode 100644 index 000000000..b7574e81d --- /dev/null +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/DynamoDbProviderSchemaException.java @@ -0,0 +1,11 @@ +package software.amazon.lambda.powertools.parameters.exception; + +/** + * Thrown when the DynamoDbProvider comes across parameter data that + * does not meet the DynamoDB parameters schema. + */ +public class DynamoDbProviderSchemaException extends RuntimeException { + public DynamoDbProviderSchemaException(String msg) { + super(msg); + } +} diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java index aaee0038c..0e5b734d6 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/DynamoDbProviderTest.java @@ -10,6 +10,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.*; import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import software.amazon.lambda.powertools.parameters.exception.DynamoDbProviderSchemaException; import java.util.HashMap; import java.util.Map; @@ -90,7 +91,7 @@ public void getValueWithMalformedRowThrows() { .item(responseData) .build()); // Act - Assertions.assertThrows(NullPointerException.class, () -> { + Assertions.assertThrows(DynamoDbProviderSchemaException.class, () -> { String value = provider.getValue(key); }); } @@ -157,7 +158,7 @@ public void getValuesWithMalformedRowThrows() { Mockito.when(client.query(queryRequestCaptor.capture())).thenReturn(response); // Assert - Assertions.assertThrows(NullPointerException.class, () -> { + Assertions.assertThrows(DynamoDbProviderSchemaException.class, () -> { // Act Map values = provider.getMultipleValues(key); });