diff --git a/CHANGES.md b/CHANGES.md index 9472b9abe36..48d7b9967b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,7 @@ Apollo 2.4.0 * [Refactor: Configuration files uniformly use Kebab style](https://github.com/apolloconfig/apollo/pull/5262) * [Feature: openapi query namespace support not fill item](https://github.com/apolloconfig/apollo/pull/5249) * [Refactor: align database ClusterName and NamespaceName fields lengths](https://github.com/apolloconfig/apollo/pull/5263) +* [Feature: Added the value length limit function for AppId-level configuration items](https://github.com/apolloconfig/apollo/pull/5264) ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index 0ab8beb38be..437b9223046 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -30,12 +30,16 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import java.util.function.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @Component public class BizConfig extends RefreshableConfig { + private final static Logger logger = LoggerFactory.getLogger(BizConfig.class); + private static final int DEFAULT_ITEM_KEY_LENGTH = 128; private static final int DEFAULT_ITEM_VALUE_LENGTH = 20000; @@ -58,6 +62,9 @@ public class BizConfig extends RefreshableConfig { private static final Gson GSON = new Gson(); + private static final Type appIdValueLengthOverrideTypeReference = + new TypeToken>() { + }.getType(); private static final Type namespaceValueLengthOverrideTypeReference = new TypeToken>() { }.getType(); @@ -106,6 +113,16 @@ public int itemValueLengthLimit() { return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_ITEM_VALUE_LENGTH); } + public Map appIdValueLengthLimitOverride() { + String appIdValueLengthOverrideString = getValue("appid.value.length.limit.override"); + return parseOverrideConfig(appIdValueLengthOverrideString, appIdValueLengthOverrideTypeReference, value -> value > 0); + } + + public Map namespaceValueLengthLimitOverride() { + String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); + return parseOverrideConfig(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference, value -> value > 0); + } + public boolean isNamespaceNumLimitEnabled() { return getBooleanProperty("namespace.num.limit.enabled", false); } @@ -128,17 +145,6 @@ public int itemNumLimit() { return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM); } - public Map namespaceValueLengthLimitOverride() { - String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); - Map namespaceValueLengthOverride = Maps.newHashMap(); - if (!Strings.isNullOrEmpty(namespaceValueLengthOverrideString)) { - namespaceValueLengthOverride = - GSON.fromJson(namespaceValueLengthOverrideString, namespaceValueLengthOverrideTypeReference); - } - - return namespaceValueLengthOverride; - } - public boolean isNamespaceLockSwitchOff() { return !getBooleanProperty("namespace.lock.switch", false); } @@ -195,15 +201,7 @@ public int releaseHistoryRetentionSize() { public Map releaseHistoryRetentionSizeOverride() { String overrideString = getValue("apollo.release-history.retention.size.override"); - Map releaseHistoryRetentionSizeOverride = Maps.newHashMap(); - if (!Strings.isNullOrEmpty(overrideString)) { - releaseHistoryRetentionSizeOverride = - GSON.fromJson(overrideString, releaseHistoryRetentionSizeOverrideTypeReference); - } - return releaseHistoryRetentionSizeOverride.entrySet() - .stream() - .filter(entry -> entry.getValue() >= 1) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return parseOverrideConfig(overrideString, releaseHistoryRetentionSizeOverrideTypeReference, value -> value > 0); } public int releaseMessageCacheScanInterval() { @@ -256,4 +254,22 @@ public boolean isAdminServiceAccessControlEnabled() { public String getAdminServiceAccessTokens() { return getValue("admin-service.access.tokens"); } + + private Map parseOverrideConfig(String configValue, Type typeReference, Predicate valueFilter) { + Map result = Maps.newHashMap(); + if (!Strings.isNullOrEmpty(configValue)) { + try { + Map parsed = GSON.fromJson(configValue, typeReference); + for (Map.Entry entry : parsed.entrySet()) { + if (entry.getValue() != null && valueFilter.test(entry.getValue())) { + result.put(entry.getKey(), entry.getValue()); + } + } + } catch (Exception e) { + logger.error("Invalid override config value: {}", configValue, e); + } + } + return Collections.unmodifiableMap(result); + } + } diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java index 0fd9199ab65..975b9814fbe 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java @@ -217,8 +217,8 @@ public Item update(Item item) { } private boolean checkItemValueLength(long namespaceId, String value) { - int limit = getItemValueLengthLimit(namespaceId); Namespace currentNamespace = namespaceService.findOne(namespaceId); + int limit = getItemValueLengthLimit(currentNamespace); if(currentNamespace != null) { Matcher m = clusterPattern.matcher(currentNamespace.getClusterName()); boolean isGray = m.matches(); @@ -235,7 +235,7 @@ private boolean checkItemValueLength(long namespaceId, String value) { private int getGrayNamespaceItemValueLengthLimit(Namespace grayNamespace, int grayNamespaceLimit) { Namespace parentNamespace = namespaceService.findParentNamespace(grayNamespace); if (parentNamespace != null) { - int parentLimit = getItemValueLengthLimit(parentNamespace.getId()); + int parentLimit = getItemValueLengthLimit(grayNamespace); if (parentLimit > grayNamespaceLimit) { return parentLimit; } @@ -257,11 +257,17 @@ private boolean checkItemType(int type) { return true; } - private int getItemValueLengthLimit(long namespaceId) { + private int getItemValueLengthLimit(Namespace namespace) { Map namespaceValueLengthOverride = bizConfig.namespaceValueLengthLimitOverride(); - if (namespaceValueLengthOverride != null && namespaceValueLengthOverride.containsKey(namespaceId)) { - return namespaceValueLengthOverride.get(namespaceId); + if (namespaceValueLengthOverride != null && namespaceValueLengthOverride.containsKey(namespace.getId())) { + return namespaceValueLengthOverride.get(namespace.getId()); } + + Map appIdValueLengthOverride = bizConfig.appIdValueLengthLimitOverride(); + if (appIdValueLengthOverride != null && appIdValueLengthOverride.containsKey(namespace.getAppId())) { + return appIdValueLengthOverride.get(namespace.getAppId()); + } + return bizConfig.itemValueLengthLimit(); } diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java index 7c7b1ea484a..c6420f187f5 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java @@ -18,13 +18,13 @@ import com.ctrip.framework.apollo.biz.repository.ServerConfigRepository; import com.ctrip.framework.apollo.biz.service.BizDBPropertySource; +import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.core.env.Environment; import org.springframework.test.util.ReflectionTestUtils; import javax.sql.DataSource; @@ -93,7 +93,7 @@ public void testReleaseHistoryRetentionSizeOverride() { int someOverrideLimit = 10; String overrideValueString = "{'a+b+c+b':10}"; when(environment.getProperty("apollo.release-history.retention.size.override")).thenReturn(overrideValueString); - int overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b"); + int overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b"); assertEquals(someOverrideLimit, overrideValue); overrideValueString = "{'a+b+c+b':0,'a+b+d+b':2}"; @@ -107,6 +107,48 @@ public void testReleaseHistoryRetentionSizeOverride() { assertEquals(0, bizConfig.releaseHistoryRetentionSizeOverride().size()); } + @Test + public void testAppIdValueLengthLimitOverride() { + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(null); + Map result = bizConfig.appIdValueLengthLimitOverride(); + assertTrue(result.isEmpty()); + + String input = "{}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + result = bizConfig.appIdValueLengthLimitOverride(); + assertTrue(result.isEmpty()); + + input = "invalid json"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + result = bizConfig.appIdValueLengthLimitOverride(); + assertTrue(result.isEmpty()); + + input = "{'appid1':555}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + int overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid1"); + assertEquals(1, bizConfig.appIdValueLengthLimitOverride().size()); + assertEquals(555, overrideValue); + + input = "{'appid1':555,'appid2':666}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + overrideValue = bizConfig.appIdValueLengthLimitOverride().get("appid2"); + assertEquals(2, bizConfig.appIdValueLengthLimitOverride().size()); + assertEquals(666, overrideValue); + + input = "{'appid1':555,'appid2':666,'appid3':0,'appid4':-1}"; + when(environment.getProperty("appid.value.length.limit.override")).thenReturn(input); + result = bizConfig.appIdValueLengthLimitOverride(); + + assertTrue(result.containsKey("appid1")); + assertTrue(result.containsKey("appid2")); + assertFalse(result.containsKey("appid3")); + assertFalse(result.containsKey("appid4")); + assertEquals(2, result.size()); + + overrideValue = result.get("appid2"); + assertEquals(666, overrideValue); + } + @Test public void testReleaseMessageNotificationBatchWithNAN() throws Exception { String someNAN = "someNAN"; diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java index 6d8003f1d58..efc4f477bdd 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemServiceTest.java @@ -16,12 +16,20 @@ */ package com.ctrip.framework.apollo.biz.service; +import static org.mockito.Mockito.when; + import com.ctrip.framework.apollo.biz.AbstractIntegrationTest; +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Item; +import com.ctrip.framework.apollo.biz.repository.ItemRepository; import com.ctrip.framework.apollo.common.dto.ItemInfoDTO; import com.ctrip.framework.apollo.common.exception.BadRequestException; +import java.util.HashMap; +import java.util.Map; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -32,18 +40,28 @@ public class ItemServiceTest extends AbstractIntegrationTest { @Autowired private ItemService itemService; + @Autowired + private ItemRepository itemRepository; + @Autowired + private NamespaceService namespaceService; + @Autowired + private AuditService auditService; + + @Mock + private BizConfig bizConfig; + + private ItemService itemService2; + + @Before + public void setUp() throws Exception { + itemService2 = new ItemService(itemRepository, namespaceService, auditService, bizConfig); + } + @Test - @Sql(scripts = "/sql/item-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = {"/sql/namespace-test.sql","/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void testSaveItem() { - Item item = new Item(); - item.setNamespaceId(3); - item.setKey("k3"); - item.setType(-1); - item.setValue("v3"); - item.setComment(""); - item.setLineNum(3); - + Item item = createItem(1L, "k3", "v3", -1); try { itemService.save(item); Assert.fail(); @@ -57,16 +75,56 @@ public void testSaveItem() { } @Test - @Sql(scripts = "/sql/item-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = {"/sql/namespace-test.sql", "/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testSaveItemWithNamespaceValueLengthLimitOverride() { + + long namespaceId = 1L; + String itemValue = "test-demo"; + + Map namespaceValueLengthOverride = new HashMap<>(); + namespaceValueLengthOverride.put(namespaceId, itemValue.length() - 1); + when(bizConfig.namespaceValueLengthLimitOverride()).thenReturn(namespaceValueLengthOverride); + when(bizConfig.itemKeyLengthLimit()).thenReturn(100); + + Item item = createItem(namespaceId, "k3", itemValue, 2); + try { + itemService2.save(item); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof BadRequestException && e.getMessage().contains("value too long")); + } + } + + @Test + @Sql(scripts = {"/sql/namespace-test.sql", "/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testSaveItemWithAppIdValueLengthLimitOverride() { + + String appId = "testApp"; + long namespaceId = 1L; + String itemValue = "test-demo"; + + Map appIdValueLengthOverride = new HashMap<>(); + appIdValueLengthOverride.put(appId, itemValue.length() - 1); + when(bizConfig.appIdValueLengthLimitOverride()).thenReturn(appIdValueLengthOverride); + when(bizConfig.itemKeyLengthLimit()).thenReturn(100); + + Item item = createItem(namespaceId, "k3", itemValue, 2); + try { + itemService2.save(item); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof BadRequestException && e.getMessage().contains("value too long")); + } + } + + @Test + @Sql(scripts = {"/sql/namespace-test.sql","/sql/item-test.sql"}, executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void testUpdateItem() { - Item item = new Item(); + Item item = createItem(1, "k1", "v1-new", 2); item.setId(9901); - item.setNamespaceId(1); - item.setKey("k1"); - item.setType(2); - item.setValue("v1-new"); - item.setComment(""); item.setLineNum(1); Item dbItem = itemService.update(item); @@ -96,4 +154,15 @@ public void testSearchItem() { } + private Item createItem(long namespaceId, String key, String value, int type) { + Item item = new Item(); + item.setNamespaceId(namespaceId); + item.setKey(key); + item.setValue(value); + item.setType(type); + item.setComment(""); + item.setLineNum(3); + return item; + } + } diff --git a/docs/en/deployment/distributed-deployment-guide.md b/docs/en/deployment/distributed-deployment-guide.md index 2331ac68d2f..91afbe7a4fb 100644 --- a/docs/en/deployment/distributed-deployment-guide.md +++ b/docs/en/deployment/distributed-deployment-guide.md @@ -1539,9 +1539,19 @@ The default configuration is 128. The default configuration is 20000. -#### 3.2.5.1 `namespace.value.length.limit.override` - Maximum length limit for namespace's configuration item value +#### 3.2.5.1 appid.value.length.limit.override - The maximum length limit of the configuration item value of the appId dimension -This configuration is used to override the `item.value.length.limit` configuration to achieve fine-grained control of the namespace's value maximum length limit, the configured value is a json format, the key of the json is the id value of the namespace in the database, the format is as follows. +This configuration is used to override the configuration of `item.value.length.limit` to control the maximum length limit of the value at the appId granularity. The configured value is in a json format, and the key of the json is appId. The format is as follows: +``` +appid.value.length.limit.override = {"appId-demo1":200,"appId-demo2":300} +``` +The above configuration specifies that the maximum length limit of the value in all namespaces under `appId-demo1` is 200, and the maximum length limit of the value in all namespaces under `appId-demo2` is 300 + +When a new namespace is created under `appId-demo1` or `appId-demo2`, it will automatically inherit the maximum length limit of the value of the namespace, unless the maximum length limit of the value of the configuration item of the namespace is overridden by `namespace.value.length.limit.override`. + +#### 3.2.5.2 `namespace.value.length.limit.override` - Maximum length limit for namespace's configuration item value + +This configuration is used to override the `item.value.length.limit` or `appid.value.length.limit.override` configuration to achieve fine-grained control of the namespace's value maximum length limit, the configured value is a json format, the key of the json is the id value of the namespace in the database, the format is as follows. ``` namespace.value.length.limit.override = {1:200,3:20} diff --git a/docs/zh/deployment/distributed-deployment-guide.md b/docs/zh/deployment/distributed-deployment-guide.md index 0b02d288a21..89473a8a0c2 100644 --- a/docs/zh/deployment/distributed-deployment-guide.md +++ b/docs/zh/deployment/distributed-deployment-guide.md @@ -1481,9 +1481,18 @@ http://5.5.5.5:8080/eureka/,http://6.6.6.6:8080/eureka/ 默认配置是20000。 -#### 3.2.5.1 namespace.value.length.limit.override - namespace 的配置项 value 最大长度限制 +#### 3.2.5.1 appid.value.length.limit.override - appId 维度的配置项 value 最大长度限制 +此配置用来覆盖 `item.value.length.limit` 的配置，做到控制 appId 粒度下的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 appId，格式如下： +``` +appid.value.length.limit.override = {"appId-demo1":200,"appId-demo2":300} +``` +以上配置指定了 `appId-demo1` 下的所有 namespace 中的 value 最大长度限制为 200，`appId-demo2` 下的所有 namespace 中的 value 最大长度限制为 300 + +当 `appId-demo1` 或 `appId-demo2` 下新建的 namespace 时，会自动继承该 namespace 的 value 最大长度限制，除非该 namespace 的配置项 value 最大长度限制被 `namespace.value.length.limit.override` 覆盖。 + +#### 3.2.5.2 namespace.value.length.limit.override - namespace 的配置项 value 最大长度限制 -此配置用来覆盖 `item.value.length.limit` 的配置，做到细粒度控制 namespace 的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 namespace 在数据库中的 id 值，格式如下： +此配置用来覆盖 `item.value.length.limit` 或者 `appid.value.length.limit.override` 的配置，做到细粒度控制 namespace 的 value 最大长度限制，配置的值是一个 json 格式，json 的 key 为 namespace 在数据库中的 id 值，格式如下： ``` namespace.value.length.limit.override = {1:200,3:20} ```