From 12a5a67dcab08f6114b16d4973c176193f359080 Mon Sep 17 00:00:00 2001 From: yangzl Date: Sun, 27 Oct 2024 15:48:22 +0800 Subject: [PATCH 1/7] feat(biz): Added the value length limit function for AppId-level configuration items --- .../apollo/biz/config/BizConfig.java | 36 +++++--- .../apollo/biz/service/ItemService.java | 16 ++-- .../apollo/biz/config/BizConfigTest.java | 39 +++++++- .../apollo/biz/service/ItemServiceTest.java | 88 ++++++++++++++++++- .../distributed-deployment-guide.md | 14 ++- .../distributed-deployment-guide.md | 13 ++- 6 files changed, 181 insertions(+), 25 deletions(-) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index 0ab8beb38be..1e0b0245899 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -58,6 +58,9 @@ public class BizConfig extends RefreshableConfig { private static final Gson GSON = new Gson(); + private static final Type appIdValueLengthOverrideTypeReference = + new TypeToken>() { + }.getType(); private static final Type namespaceValueLengthOverrideTypeReference = new TypeToken>() { }.getType(); @@ -106,6 +109,28 @@ public int itemValueLengthLimit() { return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_ITEM_VALUE_LENGTH); } + public Map appIdValueLengthLimitOverride() { + String appIdValueLengthOverrideString = getValue("appid.value.length.limit.override"); + Map appIdValueLengthOverride = Maps.newHashMap(); + if (!Strings.isNullOrEmpty(appIdValueLengthOverrideString)) { + appIdValueLengthOverride = + GSON.fromJson(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference); + } + + return appIdValueLengthOverride; + } + + public Map namespaceValueLengthLimitOverride() { + String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); + Map namespaceValueLengthOverride = Maps.newHashMap(); + if (!Strings.isNullOrEmpty(namespaceValueLengthOverrideString)) { + namespaceValueLengthOverride = + GSON.fromJson(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference); + } + + return namespaceValueLengthOverride; + } + public boolean isNamespaceNumLimitEnabled() { return getBooleanProperty("namespace.num.limit.enabled", false); } @@ -128,17 +153,6 @@ public int itemNumLimit() { return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM); } - public Map namespaceValueLengthLimitOverride() { - String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); - Map namespaceValueLengthOverride = Maps.newHashMap(); - if (!Strings.isNullOrEmpty(namespaceValueLengthOverrideString)) { - namespaceValueLengthOverride = - GSON.fromJson(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference); - } - - return namespaceValueLengthOverride; - } - public boolean isNamespaceLockSwitchOff() { return !getBooleanProperty("namespace.lock.switch", false); } diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java index 0fd9199ab65..975b9814fbe 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java @@ -217,8 +217,8 @@ public Item update(Item item) { } private boolean checkItemValueLength(long namespaceId, String value) { - int limit = getItemValueLengthLimit(namespaceId); Namespace currentNamespace = namespaceService.findOne(namespaceId); + int limit = getItemValueLengthLimit(currentNamespace); if(currentNamespace != null) { Matcher m = clusterPattern.matcher(currentNamespace.getClusterName()); boolean isGray = m.matches(); @@ -235,7 +235,7 @@ private boolean checkItemValueLength(long namespaceId, String value) { private int getGrayNamespaceItemValueLengthLimit(Namespace grayNamespace, int grayNamespaceLimit) { Namespace parentNamespace = namespaceService.findParentNamespace(grayNamespace); if (parentNamespace != null) { - int parentLimit = getItemValueLengthLimit(parentNamespace.getId()); + int parentLimit = getItemValueLengthLimit(grayNamespace); if (parentLimit > grayNamespaceLimit) { return parentLimit; } @@ -257,11 +257,17 @@ private boolean checkItemType(int type) { return true; } - private int getItemValueLengthLimit(long namespaceId) { + private int getItemValueLengthLimit(Namespace namespace) { Map namespaceValueLengthOverride = bizConfig.namespaceValueLengthLimitOverride(); - if (namespaceValueLengthOverride != null && namespaceValueLengthOverride.containsKey(namespaceId)) { - return namespaceValueLengthOverride.get(namespaceId); + if (namespaceValueLengthOverride != null && namespaceValueLengthOverride.containsKey(namespace.getId())) { + return namespaceValueLengthOverride.get(namespace.getId()); } + + Map appIdValueLengthOverride = bizConfig.appIdValueLengthLimitOverride(); + if (appIdValueLengthOverride != null && appIdValueLengthOverride.containsKey(namespace.getAppId())) { + return appIdValueLengthOverride.get(namespace.getAppId()); + } + return bizConfig.itemValueLengthLimit(); } diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java index 7c7b1ea484a..e6f5a386e84 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java @@ -18,13 +18,15 @@ import com.ctrip.framework.apollo.biz.repository.ServerConfigRepository; import com.ctrip.framework.apollo.biz.service.BizDBPropertySource; +import com.google.gson.JsonSyntaxException; +import java.util.Map; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.core.env.Environment; import org.springframework.test.util.ReflectionTestUtils; import javax.sql.DataSource; @@ -93,7 +95,7 @@ public void testReleaseHistoryRetentionSizeOverride() { int someOverrideLimit = 10; String overrideValueString = "{'a+b+c+b':10}"; when(environment.getProperty("apollo.release-history.retention.size.override")).thenReturn(overrideValueString); - int overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b"); + int overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b"); assertEquals(someOverrideLimit, overrideValue); overrideValueString = "{'a+b+c+b':0,'a+b+d+b':2}"; @@ -107,6 +109,39 @@ public void testReleaseHistoryRetentionSizeOverride() { assertEquals(0, bizConfig.releaseHistoryRetentionSizeOverride().size()); } + @Test + public void testAppIdValueLengthLimitOverride() { + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(null); + Map result = bizConfig.appIdValueLengthLimitOverride(); + assertTrue(result.isEmpty()); + + String input = "{}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + result = bizConfig.appIdValueLengthLimitOverride(); + assertTrue(result.isEmpty()); + + try { + input = "invalid json"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + bizConfig.appIdValueLengthLimitOverride(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof JsonSyntaxException); + } + + input = "{'appid1':555}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + int overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid1"); + assertEquals(1, bizConfig.appIdValueLengthLimitOverride().size()); + assertEquals(555, overrideValue); + + input = "{'appid1':555,'appid2':666}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid2"); + assertEquals(2, bizConfig.appIdValueLengthLimitOverride().size()); + assertEquals(666, overrideValue); + } + @Test public void testReleaseMessageNotificationBatchWithNAN() throws Exception { String someNAN = "someNAN"; diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java index 6d8003f1d58..894fab59bbd 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java @@ -16,12 +16,20 @@ */ package com.ctrip.framework.apollo.biz.service; +import static org.mockito.Mockito.when; + import com.ctrip.framework.apollo.biz.AbstractIntegrationTest; +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Item; +import com.ctrip.framework.apollo.biz.repository.ItemRepository; import com.ctrip.framework.apollo.common.dto.ItemInfoDTO; import com.ctrip.framework.apollo.common.exception.BadRequestException; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -32,12 +40,29 @@ public class ItemServiceTest extends AbstractIntegrationTest { @Autowired private ItemService itemService; + @Autowired + private ItemRepository itemRepository; + @Autowired + private NamespaceService namespaceService; + @Autowired + private AuditService auditService; + + @Mock + private BizConfig bizConfig; + + private ItemService itemService2; + + @Before + public void setUp() throws Exception { + itemService2 = new ItemService(itemRepository, namespaceService, auditService, bizConfig); + } + @Test - @Sql(scripts = "/sql/item-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = {"/sql/namespace-test.sql","/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void testSaveItem() { Item item = new Item(); - item.setNamespaceId(3); + item.setNamespaceId(1); item.setKey("k3"); item.setType(-1); item.setValue("v3"); @@ -57,7 +82,64 @@ public void testSaveItem() { } @Test - @Sql(scripts = "/sql/item-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = {"/sql/namespace-test.sql", "/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testSaveItemWithNamespaceValueLengthLimitOverride() { + + long namespaceId = 1L; + String itemValue = "test-demo"; + + Map namespaceValueLengthOverride = new HashMap<>(); + namespaceValueLengthOverride.put(namespaceId, itemValue.length() - 1); + when(bizConfig.namespaceValueLengthLimitOverride()).thenReturn(namespaceValueLengthOverride); + when(bizConfig.itemKeyLengthLimit()).thenReturn(100); + + Item item = new Item(); + item.setNamespaceId(namespaceId); + item.setKey("k3"); + item.setType(2); + item.setValue(itemValue); + item.setComment(""); + item.setLineNum(3); + try { + itemService2.save(item); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof BadRequestException && e.getMessage().contains("value too long")); + } + } + + @Test + @Sql(scripts = {"/sql/namespace-test.sql", "/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testSaveItemWithAppIdValueLengthLimitOverride() { + + String appId = "testApp"; + long namespaceId = 1L; + String itemValue = "test-demo"; + + Map appIdValueLengthOverride = new HashMap<>(); + appIdValueLengthOverride.put(appId, itemValue.length() - 1); + when(bizConfig.appIdValueLengthLimitOverride()).thenReturn(appIdValueLengthOverride); + when(bizConfig.itemKeyLengthLimit()).thenReturn(100); + + Item item = new Item(); + item.setNamespaceId(namespaceId); + item.setKey("k3"); + item.setType(2); + item.setValue(itemValue); + item.setComment(""); + item.setLineNum(3); + try { + itemService2.save(item); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof BadRequestException && e.getMessage().contains("value too long")); + } + } + + @Test + @Sql(scripts = {"/sql/namespace-test.sql","/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void testUpdateItem() { Item item = new Item(); diff --git a/docs/en/deployment/distributed-deployment-guide.md b/docs/en/deployment/distributed-deployment-guide.md index ff820d62869..c6c8ff4493e 100644 --- a/docs/en/deployment/distributed-deployment-guide.md +++ b/docs/en/deployment/distributed-deployment-guide.md @@ -1539,9 +1539,19 @@ The default configuration is 128. The default configuration is 20000. -#### 3.2.5.1 `namespace.value.length.limit.override` - Maximum length limit for namespace's configuration item value +#### 3.2.5.1 appid.value.length.limit.override - The maximum length limit of the configuration item value of the appId dimension -This configuration is used to override the `item.value.length.limit` configuration to achieve fine-grained control of the namespace's value maximum length limit, the configured value is a json format, the key of the json is the id value of the namespace in the database, the format is as follows. +This configuration is used to override the configuration of `item.value.length.limit` to control the maximum length limit of the value at the appId granularity. The configured value is in a json format, and the key of the json is appId. The format is as follows: +``` +appid.value.length.limit.override = {appId-demo1:200,aappId-demo2:300} +``` +The above configuration specifies that the maximum length limit of the value in all namespaces under `appId-demo1` is 200, and the maximum length limit of the value in all namespaces under `appId-demo2` is 300 + +When a new namespace is created under `appId-demo1` or `appId-demo2`, it will automatically inherit the maximum length limit of the value of the namespace, unless the maximum length limit of the value of the configuration item of the namespace is overridden by `namespace.value.length.limit.override`. + +#### 3.2.5.2 `namespace.value.length.limit.override` - Maximum length limit for namespace's configuration item value + +This configuration is used to override the `item.value.length.limit` or `appid.value.length.limit.override` configuration to achieve fine-grained control of the namespace's value maximum length limit, the configured value is a json format, the key of the json is the id value of the namespace in the database, the format is as follows. ``` namespace.value.length.limit.override = {1:200,3:20} diff --git a/docs/zh/deployment/distributed-deployment-guide.md b/docs/zh/deployment/distributed-deployment-guide.md index 85779395a0c..a33bbc24aca 100644 --- a/docs/zh/deployment/distributed-deployment-guide.md +++ b/docs/zh/deployment/distributed-deployment-guide.md @@ -1481,9 +1481,18 @@ http://5.5.5.5:8080/eureka/,http://6.6.6.6:8080/eureka/ 默认配置是20000。 -#### 3.2.5.1 namespace.value.length.limit.override - namespace 的配置项 value 最大长度限制 +#### 3.2.5.1 appid.value.length.limit.override - appId 维度的配置项 value 最大长度限制 +此配置用来覆盖 `item.value.length.limit` 的配置，做到控制 appId 粒度下的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 appId，格式如下： +``` +appid.value.length.limit.override = {appId-demo1:200,aappId-demo2:300} +``` +以上配置指定了 `appId-demo1` 下的所有 namespace 中的 value 最大长度限制为 200，`appId-demo2` 下的所有 namespace 中的 value 最大长度限制为 300 + +当 `appId-demo1` 或 `appId-demo2` 下新建的 namespace 时，会自动继承该 namespace 的 value 最大长度限制，除非该 namespace 的配置项 value 最大长度限制被 `namespace.value.length.limit.override` 覆盖。 + +#### 3.2.5.2 namespace.value.length.limit.override - namespace 的配置项 value 最大长度限制 -此配置用来覆盖 `item.value.length.limit` 的配置，做到细粒度控制 namespace 的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 namespace 在数据库中的 id 值，格式如下： +此配置用来覆盖 `item.value.length.limit` 或者 `appid.value.length.limit.override` 的配置，做到细粒度控制 namespace 的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 namespace 在数据库中的 id 值，格式如下： ``` namespace.value.length.limit.override = {1:200,3:20} ``` From 6ab0caa721b58e9a4539e87802313ceaa3a94bd4 Mon Sep 17 00:00:00 2001 From: yangzl Date: Sun, 27 Oct 2024 15:52:03 +0800 Subject: [PATCH 2/7] =?UTF-8?q?fix=EF=BC=9Aadd=20CHANGES.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 81d5936d2bf..d33203beb32 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,7 @@ Apollo 2.4.0 * [Feature add limit for items count per namespace](https://github.com/apolloconfig/apollo/pull/5227) * [Feature: Add ConfigService cache record stats function](https://github.com/apolloconfig/apollo/pull/5247) * [RefreshAdminServerAddressTask supports dynamic configuration of time interval](https://github.com/apolloconfig/apollo/pull/5248) +* [Feature: Added the value length limit function for AppId-level configuration items](https://github.com/apolloconfig/apollo/pull/5264) ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1) From c380c9106a22f64d9350f002f838c65e5305db00 Mon Sep 17 00:00:00 2001 From: yangzl Date: Sun, 27 Oct 2024 17:55:05 +0800 Subject: [PATCH 3/7] refactor(biz): Refactor configuration parsing logic and optimize test cases --- .../apollo/biz/config/BizConfig.java | 46 +++++++++---------- .../apollo/biz/config/BizConfigTest.java | 21 +++++---- .../apollo/biz/service/ItemServiceTest.java | 43 ++++++----------- 3 files changed, 49 insertions(+), 61 deletions(-) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index 1e0b0245899..cca14620c09 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -31,11 +31,15 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @Component public class BizConfig extends RefreshableConfig { + private final static Logger logger = LoggerFactory.getLogger(BizConfig.class); + private static final int DEFAULT_ITEM_KEY_LENGTH = 128; private static final int DEFAULT_ITEM_VALUE_LENGTH = 20000; @@ -111,24 +115,12 @@ public int itemValueLengthLimit() { public Map appIdValueLengthLimitOverride() { String appIdValueLengthOverrideString = getValue("appid.value.length.limit.override"); - Map appIdValueLengthOverride = Maps.newHashMap(); - if (!Strings.isNullOrEmpty(appIdValueLengthOverrideString)) { - appIdValueLengthOverride = - GSON.fromJson(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference); - } - - return appIdValueLengthOverride; + return parseOverrideConfig(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference); } public Map namespaceValueLengthLimitOverride() { String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); - Map namespaceValueLengthOverride = Maps.newHashMap(); - if (!Strings.isNullOrEmpty(namespaceValueLengthOverrideString)) { - namespaceValueLengthOverride = - GSON.fromJson(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference); - } - - return namespaceValueLengthOverride; + return parseOverrideConfig(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference); } public boolean isNamespaceNumLimitEnabled() { @@ -209,15 +201,7 @@ public int releaseHistoryRetentionSize() { public Map releaseHistoryRetentionSizeOverride() { String overrideString = getValue("apollo.release-history.retention.size.override"); - Map releaseHistoryRetentionSizeOverride = Maps.newHashMap(); - if (!Strings.isNullOrEmpty(overrideString)) { - releaseHistoryRetentionSizeOverride = - GSON.fromJson(overrideString, releaseHistoryRetentionSizeOverrideTypeReference); - } - return releaseHistoryRetentionSizeOverride.entrySet() - .stream() - .filter(entry -> entry.getValue() >= 1) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return parseOverrideConfig(overrideString, releaseHistoryRetentionSizeOverrideTypeReference); } public int releaseMessageCacheScanInterval() { @@ -270,4 +254,20 @@ public boolean isAdminServiceAccessControlEnabled() { public String getAdminServiceAccessTokens() { return getValue("admin-service.access.tokens"); } + + private Map parseOverrideConfig(String configValue, Type typeReference) { + Map result = Maps.newHashMap(); + if (!Strings.isNullOrEmpty(configValue)) { + try { + Map parsed = GSON.fromJson(configValue, typeReference); + result = parsed.entrySet().stream() + .filter(e -> e.getValue() != null && ((Integer) e.getValue()) > 0) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } catch (Exception e) { + logger.error("Invalid override config value: {}", configValue, e); + } + } + return Collections.unmodifiableMap(result); + } + } diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java index e6f5a386e84..32edf67fb31 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java @@ -18,9 +18,7 @@ import com.ctrip.framework.apollo.biz.repository.ServerConfigRepository; import com.ctrip.framework.apollo.biz.service.BizDBPropertySource; -import com.google.gson.JsonSyntaxException; import java.util.Map; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -120,14 +118,10 @@ public void testAppIdValueLengthLimitOverride() { result = bizConfig.appIdValueLengthLimitOverride(); assertTrue(result.isEmpty()); - try { - input = "invalid json"; - when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); - bizConfig.appIdValueLengthLimitOverride(); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e instanceof JsonSyntaxException); - } + input = "invalid json"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + result = bizConfig.appIdValueLengthLimitOverride(); + assertTrue(result.isEmpty()); input = "{'appid1':555}"; when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); @@ -140,6 +134,13 @@ public void testAppIdValueLengthLimitOverride() { overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid2"); assertEquals(2, bizConfig.appIdValueLengthLimitOverride().size()); assertEquals(666, overrideValue); + + input = "{'appid1':555,'appid2':666,'appid3':0,'appid4':-1}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid2"); + assertEquals(2, bizConfig.appIdValueLengthLimitOverride().size()); + assertEquals(666, overrideValue); + } @Test diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java index 894fab59bbd..efc4f477bdd 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java @@ -61,14 +61,7 @@ public void setUp() throws Exception { @Sql(scripts = {"/sql/namespace-test.sql","/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void testSaveItem() { - Item item = new Item(); - item.setNamespaceId(1); - item.setKey("k3"); - item.setType(-1); - item.setValue("v3"); - item.setComment(""); - item.setLineNum(3); - + Item item = createItem(1L, "k3", "v3", -1); try { itemService.save(item); Assert.fail(); @@ -94,13 +87,7 @@ public void testSaveItemWithNamespaceValueLengthLimitOverride() { when(bizConfig.namespaceValueLengthLimitOverride()).thenReturn(namespaceValueLengthOverride); when(bizConfig.itemKeyLengthLimit()).thenReturn(100); - Item item = new Item(); - item.setNamespaceId(namespaceId); - item.setKey("k3"); - item.setType(2); - item.setValue(itemValue); - item.setComment(""); - item.setLineNum(3); + Item item = createItem(namespaceId, "k3", itemValue, 2); try { itemService2.save(item); Assert.fail(); @@ -123,13 +110,7 @@ public void testSaveItemWithAppIdValueLengthLimitOverride() { when(bizConfig.appIdValueLengthLimitOverride()).thenReturn(appIdValueLengthOverride); when(bizConfig.itemKeyLengthLimit()).thenReturn(100); - Item item = new Item(); - item.setNamespaceId(namespaceId); - item.setKey("k3"); - item.setType(2); - item.setValue(itemValue); - item.setComment(""); - item.setLineNum(3); + Item item = createItem(namespaceId, "k3", itemValue, 2); try { itemService2.save(item); Assert.fail(); @@ -142,13 +123,8 @@ public void testSaveItemWithAppIdValueLengthLimitOverride() { @Sql(scripts = {"/sql/namespace-test.sql","/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void testUpdateItem() { - Item item = new Item(); + Item item = createItem(1, "k1", "v1-new", 2); item.setId(9901); - item.setNamespaceId(1); - item.setKey("k1"); - item.setType(2); - item.setValue("v1-new"); - item.setComment(""); item.setLineNum(1); Item dbItem = itemService.update(item); @@ -178,4 +154,15 @@ public void testSearchItem() { } + private Item createItem(long namespaceId, String key, String value, int type) { + Item item = new Item(); + item.setNamespaceId(namespaceId); + item.setKey(key); + item.setValue(value); + item.setType(type); + item.setComment(""); + item.setLineNum(3); + return item; + } + } From 2bb1e5f651777bd147a0f7c1bcb3f1072fc24d6a Mon Sep 17 00:00:00 2001 From: yangzl Date: Sun, 27 Oct 2024 18:10:25 +0800 Subject: [PATCH 4/7] test(biz): Optimize appIdValueLengthLimitOverride test case --- .../framework/apollo/biz/config/BizConfigTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java index 32edf67fb31..c6420f187f5 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java @@ -137,10 +137,16 @@ public void testAppIdValueLengthLimitOverride() { input = "{'appid1':555,'appid2':666,'appid3':0,'appid4':-1}"; when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); - overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid2"); - assertEquals(2, bizConfig.appIdValueLengthLimitOverride().size()); - assertEquals(666, overrideValue); + result = bizConfig.appIdValueLengthLimitOverride(); + + assertTrue(result.containsKey("appid1")); + assertTrue(result.containsKey("appid2")); + assertFalse(result.containsKey("appid3")); + assertFalse(result.containsKey("appid4")); + assertEquals(2, result.size()); + overrideValue = result.get("appid2"); + assertEquals(666, overrideValue); } @Test From cd70ba7ac0fd7f881dd7343f1e9c51f43029a11f Mon Sep 17 00:00:00 2001 From: yangzl Date: Sun, 27 Oct 2024 18:16:50 +0800 Subject: [PATCH 5/7] refactor: Triggering a build --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index d33203beb32..649fcb364e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,5 +17,6 @@ Apollo 2.4.0 * [RefreshAdminServerAddressTask supports dynamic configuration of time interval](https://github.com/apolloconfig/apollo/pull/5248) * [Feature: Added the value length limit function for AppId-level configuration items](https://github.com/apolloconfig/apollo/pull/5264) + ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1) From 2a0aa08e92df92e741cf20f7a4bf955b24013f52 Mon Sep 17 00:00:00 2001 From: yangzl Date: Sun, 27 Oct 2024 18:35:21 +0800 Subject: [PATCH 6/7] refactor: Triggering a build2 --- CHANGES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 649fcb364e4..d33203beb32 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,5 @@ Apollo 2.4.0 * [RefreshAdminServerAddressTask supports dynamic configuration of time interval](https://github.com/apolloconfig/apollo/pull/5248) * [Feature: Added the value length limit function for AppId-level configuration items](https://github.com/apolloconfig/apollo/pull/5264) - ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1) From f4b4e9b546e208b4956dfa5642bcb70f8cde7a31 Mon Sep 17 00:00:00 2001 From: yangzl Date: Sat, 2 Nov 2024 21:01:02 +0800 Subject: [PATCH 7/7] fix(apollo-biz): Fix the parsing logic of configuration override values and add value validity verification --- .../framework/apollo/biz/config/BizConfig.java | 18 ++++++++++-------- .../deployment/distributed-deployment-guide.md | 2 +- .../deployment/distributed-deployment-guide.md | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index cca14620c09..437b9223046 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import java.util.function.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -115,12 +115,12 @@ public int itemValueLengthLimit() { public Map appIdValueLengthLimitOverride() { String appIdValueLengthOverrideString = getValue("appid.value.length.limit.override"); - return parseOverrideConfig(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference); + return parseOverrideConfig(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference, value -> value > 0); } public Map namespaceValueLengthLimitOverride() { String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); - return parseOverrideConfig(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference); + return parseOverrideConfig(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference, value -> value > 0); } public boolean isNamespaceNumLimitEnabled() { @@ -201,7 +201,7 @@ public int releaseHistoryRetentionSize() { public Map releaseHistoryRetentionSizeOverride() { String overrideString = getValue("apollo.release-history.retention.size.override"); - return parseOverrideConfig(overrideString, releaseHistoryRetentionSizeOverrideTypeReference); + return parseOverrideConfig(overrideString, releaseHistoryRetentionSizeOverrideTypeReference, value -> value > 0); } public int releaseMessageCacheScanInterval() { @@ -255,14 +255,16 @@ public String getAdminServiceAccessTokens() { return getValue("admin-service.access.tokens"); } - private Map parseOverrideConfig(String configValue, Type typeReference) { + private Map parseOverrideConfig(String configValue, Type typeReference, Predicate valueFilter) { Map result = Maps.newHashMap(); if (!Strings.isNullOrEmpty(configValue)) { try { Map parsed = GSON.fromJson(configValue, typeReference); - result = parsed.entrySet().stream() - .filter(e -> e.getValue() != null && ((Integer) e.getValue()) > 0) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + for (Map.Entry entry : parsed.entrySet()) { + if (entry.getValue() != null && valueFilter.test(entry.getValue())) { + result.put(entry.getKey(), entry.getValue()); + } + } } catch (Exception e) { logger.error("Invalid override config value: {}", configValue, e); } diff --git a/docs/en/deployment/distributed-deployment-guide.md b/docs/en/deployment/distributed-deployment-guide.md index aa3296995ef..91afbe7a4fb 100644 --- a/docs/en/deployment/distributed-deployment-guide.md +++ b/docs/en/deployment/distributed-deployment-guide.md @@ -1543,7 +1543,7 @@ The default configuration is 20000. This configuration is used to override the configuration of `item.value.length.limit` to control the maximum length limit of the value at the appId granularity. The configured value is in a json format, and the key of the json is appId. The format is as follows: ``` -appid.value.length.limit.override = {appId-demo1:200,aappId-demo2:300} +appid.value.length.limit.override = {"appId-demo1":200,"appId-demo2":300} ``` The above configuration specifies that the maximum length limit of the value in all namespaces under `appId-demo1` is 200, and the maximum length limit of the value in all namespaces under `appId-demo2` is 300 diff --git a/docs/zh/deployment/distributed-deployment-guide.md b/docs/zh/deployment/distributed-deployment-guide.md index 284d234a0d0..89473a8a0c2 100644 --- a/docs/zh/deployment/distributed-deployment-guide.md +++ b/docs/zh/deployment/distributed-deployment-guide.md @@ -1484,7 +1484,7 @@ http://5.5.5.5:8080/eureka/,http://6.6.6.6:8080/eureka/ #### 3.2.5.1 appid.value.length.limit.override - appId 维度的配置项 value 最大长度限制 此配置用来覆盖 `item.value.length.limit` 的配置，做到控制 appId 粒度下的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 appId，格式如下： ``` -appid.value.length.limit.override = {appId-demo1:200,aappId-demo2:300} +appid.value.length.limit.override = {"appId-demo1":200,"appId-demo2":300} ``` 以上配置指定了 `appId-demo1` 下的所有 namespace 中的 value 最大长度限制为 200，`appId-demo2` 下的所有 namespace 中的 value 最大长度限制为 300