diff --git a/CHANGES.md b/CHANGES.md index f151343137f..167134551db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,6 @@ Apollo 2.4.0 * [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240) * [Feature: Add limit and whitelist for namespace count per appid+cluster](https://github.com/apolloconfig/apollo/pull/5228) * [Feature support the observe status access-key for pre-check and logging only](https://github.com/apolloconfig/apollo/pull/5236) - +* [Feature add limit for items count per namespace](https://github.com/apolloconfig/apollo/pull/5227) ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1) diff --git a/apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java b/apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java index f628748d2fa..db1c3e1d1cb 100644 --- a/apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java +++ b/apollo-adminservice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java @@ -17,6 +17,7 @@ package com.ctrip.framework.apollo.adminservice.controller; import com.ctrip.framework.apollo.adminservice.aop.PreAcquireNamespaceLock; +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Commit; import com.ctrip.framework.apollo.biz.entity.Item; import com.ctrip.framework.apollo.biz.entity.Namespace; @@ -58,13 +59,14 @@ public class ItemController { private final NamespaceService namespaceService; private final CommitService commitService; private final ReleaseService releaseService; + private final BizConfig bizConfig; - - public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService) { + public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService, final BizConfig bizConfig) { this.itemService = itemService; this.namespaceService = namespaceService; this.commitService = commitService; this.releaseService = releaseService; + this.bizConfig = bizConfig; } @PreAcquireNamespaceLock @@ -78,6 +80,14 @@ public ItemDTO create(@PathVariable("appId") String appId, if (managedEntity != null) { throw BadRequestException.itemAlreadyExists(entity.getKey()); } + + if (bizConfig.isItemNumLimitEnabled()) { + int itemCount = itemService.findNonEmptyItemCount(entity.getNamespaceId()); + if (itemCount >= bizConfig.itemNumLimit()) { + throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + "."); + } + } + entity = itemService.save(entity); dto = BeanUtils.transform(ItemDTO.class, entity); commitService.createCommit(appId, clusterName, namespaceName, new ConfigChangeContentBuilder().createItem(entity).build(), diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index fe26e81bc45..37fe1ed0c6f 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -41,6 +41,8 @@ public class BizConfig extends RefreshableConfig { private static final int DEFAULT_MAX_NAMESPACE_NUM = 200; + private static final int DEFAULT_MAX_ITEM_NUM = 1000; + private static final int DEFAULT_APPNAMESPACE_CACHE_REBUILD_INTERVAL = 60; //60s private static final int DEFAULT_GRAY_RELEASE_RULE_SCAN_INTERVAL = 60; //60s private static final int DEFAULT_APPNAMESPACE_CACHE_SCAN_INTERVAL = 1; //1s @@ -117,6 +119,15 @@ public Set namespaceNumLimitWhite() { return Sets.newHashSet(getArrayProperty("namespace.num.limit.white", new String[0])); } + public boolean isItemNumLimitEnabled() { + return getBooleanProperty("item.num.limit.enabled", false); + } + + public int itemNumLimit() { + int limit = getIntProperty("item.num.limit", DEFAULT_MAX_ITEM_NUM); + return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM); + } + public Map namespaceValueLengthLimitOverride() { String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override"); Map namespaceValueLengthOverride = Maps.newHashMap(); diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java index c866b902814..55da734b810 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ItemRepository.java @@ -64,4 +64,7 @@ public interface ItemRepository extends PagingAndSortingRepository { @Query("update Item set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?2 where NamespaceId = ?1 and IsDeleted = false") int deleteByNamespaceId(long namespaceId, String operator); + @Query("select count(*) from Item where namespaceId = :namespaceId and key <>''") + int countByNamespaceIdAndFilterKeyEmpty(@Param("namespaceId") long namespaceId); + } diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java index 3c0d23c1b62..0fd9199ab65 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java @@ -135,6 +135,10 @@ public List findItemsModifiedAfterDate(long namespaceId, Date date) { return itemRepository.findByNamespaceIdAndDataChangeLastModifiedTimeGreaterThan(namespaceId, date); } + public int findNonEmptyItemCount(long namespaceId) { + return itemRepository.countByNamespaceIdAndFilterKeyEmpty(namespaceId); + } + public Page findItemsByKey(String key, Pageable pageable) { return itemRepository.findByKey(key, pageable); } diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java index f1559d2643d..dca6089de10 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemSetService.java @@ -16,6 +16,7 @@ */ package com.ctrip.framework.apollo.biz.service; +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Audit; import com.ctrip.framework.apollo.biz.entity.Item; import com.ctrip.framework.apollo.biz.entity.Namespace; @@ -25,6 +26,7 @@ import com.ctrip.framework.apollo.common.exception.BadRequestException; import com.ctrip.framework.apollo.common.exception.NotFoundException; import com.ctrip.framework.apollo.common.utils.BeanUtils; +import com.ctrip.framework.apollo.core.utils.StringUtils; import java.util.List; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -38,20 +40,23 @@ public class ItemSetService { private final CommitService commitService; private final ItemService itemService; private final NamespaceService namespaceService; + private final BizConfig bizConfig; public ItemSetService( final AuditService auditService, final CommitService commitService, final ItemService itemService, - final NamespaceService namespaceService) { + final NamespaceService namespaceService, + final BizConfig bizConfig) { this.auditService = auditService; this.commitService = commitService; this.itemService = itemService; this.namespaceService = namespaceService; + this.bizConfig = bizConfig; } @Transactional - public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets){ + public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets) { return updateSet(namespace.getAppId(), namespace.getClusterName(), namespace.getNamespaceName(), changeSets); } @@ -64,6 +69,16 @@ public ItemChangeSets updateSet(String appId, String clusterName, throw NotFoundException.namespaceNotFound(appId, clusterName, namespaceName); } + if (bizConfig.isItemNumLimitEnabled()) { + int itemCount = itemService.findNonEmptyItemCount(namespace.getId()); + int createItemCount = (int) changeSet.getCreateItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count(); + int deleteItemCount = (int) changeSet.getDeleteItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count(); + itemCount = itemCount + createItemCount - deleteItemCount; + if (itemCount > bizConfig.itemNumLimit()) { + throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + "."); + } + } + String operator = changeSet.getDataChangeLastModifiedBy(); ConfigChangeContentBuilder configChangeContentBuilder = new ConfigChangeContentBuilder(); @@ -84,7 +99,7 @@ public ItemChangeSets updateSet(String appId, String clusterName, if (configChangeContentBuilder.hasContent()) { commitService.createCommit(appId, clusterName, namespaceName, configChangeContentBuilder.build(), - changeSet.getDataChangeLastModifiedBy()); + changeSet.getDataChangeLastModifiedBy()); } return changeSet; diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemSetServiceTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemSetServiceTest.java new file mode 100644 index 00000000000..e1dce5a16ba --- /dev/null +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ItemSetServiceTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.biz.service; + +import static org.mockito.Mockito.when; + +import com.ctrip.framework.apollo.biz.AbstractIntegrationTest; +import com.ctrip.framework.apollo.biz.config.BizConfig; +import com.ctrip.framework.apollo.biz.entity.Item; +import com.ctrip.framework.apollo.biz.entity.Namespace; +import com.ctrip.framework.apollo.common.dto.ItemChangeSets; +import com.ctrip.framework.apollo.common.dto.ItemDTO; +import com.ctrip.framework.apollo.common.exception.BadRequestException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.jdbc.Sql; + +public class ItemSetServiceTest extends AbstractIntegrationTest { + + @MockBean + private BizConfig bizConfig; + + @Autowired + private ItemService itemService; + @Autowired + private NamespaceService namespaceService; + + @Autowired + private ItemSetService itemSetService; + + @Test + @Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testUpdateSetWithoutItemNumLimit() { + + when(bizConfig.itemKeyLengthLimit()).thenReturn(128); + when(bizConfig.itemValueLengthLimit()).thenReturn(20000); + + when(bizConfig.isItemNumLimitEnabled()).thenReturn(false); + when(bizConfig.itemNumLimit()).thenReturn(5); + + Namespace namespace = namespaceService.findOne(1L); + + ItemChangeSets changeSets = new ItemChangeSets(); + changeSets.addCreateItem(buildNormalItem(0L, namespace.getId(), "k6", "v6", "test item num limit", 6)); + changeSets.addCreateItem(buildNormalItem(0L, namespace.getId(), "k7", "v7", "test item num limit", 7)); + + try { + itemSetService.updateSet(namespace, changeSets); + } catch (Exception e) { + Assert.fail(); + } + + int size = itemService.findNonEmptyItemCount(namespace.getId()); + Assert.assertEquals(7, size); + + } + + @Test + @Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testUpdateSetWithItemNumLimit() { + + when(bizConfig.itemKeyLengthLimit()).thenReturn(128); + when(bizConfig.itemValueLengthLimit()).thenReturn(20000); + + when(bizConfig.isItemNumLimitEnabled()).thenReturn(true); + when(bizConfig.itemNumLimit()).thenReturn(5); + + Namespace namespace = namespaceService.findOne(1L); + Item item9901 = itemService.findOne(9901); + Item item9902 = itemService.findOne(9902); + + ItemChangeSets changeSets = new ItemChangeSets(); + changeSets.addUpdateItem(buildNormalItem(item9901.getId(), item9901.getNamespaceId(), item9901.getKey(), item9901.getValue() + " update", item9901.getComment(), item9901.getLineNum())); + changeSets.addDeleteItem(buildNormalItem(item9902.getId(), item9902.getNamespaceId(), item9902.getKey(), item9902.getValue() + " update", item9902.getComment(), item9902.getLineNum())); + changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k6", "v6", "test item num limit", 6)); + changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k7", "v7", "test item num limit", 7)); + + try { + itemSetService.updateSet(namespace, changeSets); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof BadRequestException); + } + + int size = itemService.findNonEmptyItemCount(namespace.getId()); + Assert.assertEquals(5, size); + + } + + @Test + @Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) + public void testUpdateSetWithItemNumLimit2() { + + when(bizConfig.itemKeyLengthLimit()).thenReturn(128); + when(bizConfig.itemValueLengthLimit()).thenReturn(20000); + + when(bizConfig.isItemNumLimitEnabled()).thenReturn(true); + when(bizConfig.itemNumLimit()).thenReturn(5); + + Namespace namespace = namespaceService.findOne(1L); + Item item9901 = itemService.findOne(9901); + Item item9902 = itemService.findOne(9902); + + ItemChangeSets changeSets = new ItemChangeSets(); + changeSets.addUpdateItem(buildNormalItem(item9901.getId(), item9901.getNamespaceId(), item9901.getKey(), item9901.getValue() + " update", item9901.getComment(), item9901.getLineNum())); + changeSets.addDeleteItem(buildNormalItem(item9902.getId(), item9902.getNamespaceId(), item9902.getKey(), item9902.getValue() + " update", item9902.getComment(), item9902.getLineNum())); + changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k6", "v6", "test item num limit", 6)); + + try { + itemSetService.updateSet(namespace, changeSets); + } catch (Exception e) { + Assert.fail(); + } + + int size = itemService.findNonEmptyItemCount(namespace.getId()); + Assert.assertEquals(5, size); + + } + + + private ItemDTO buildNormalItem(Long id, Long namespaceId, String key, String value, String comment, int lineNum) { + ItemDTO item = new ItemDTO(key, value, comment, lineNum); + item.setId(id); + item.setNamespaceId(namespaceId); + return item; + } + +} diff --git a/apollo-biz/src/test/resources/sql/itemset-test.sql b/apollo-biz/src/test/resources/sql/itemset-test.sql new file mode 100644 index 00000000000..4a57e307b9d --- /dev/null +++ b/apollo-biz/src/test/resources/sql/itemset-test.sql @@ -0,0 +1,25 @@ +-- +-- Copyright 2024 Apollo Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +INSERT INTO "Namespace" (`Id`, `AppId`, `ClusterName`, `NamespaceName`, `IsDeleted`, `DataChange_CreatedBy`, `DataChange_LastModifiedBy`)VALUES(1,'testApp', 'default', 'application', 0, 'apollo', 'apollo'); + +INSERT INTO "Item" (`Id`, `NamespaceId`, "Key", "Type", "Value", `Comment`, `LineNum`) + VALUES + (9901, 1, 'k1', 0, 'v1', '', 1), + (9902, 1, 'k2', 2, 'v2', '', 2), + (9903, 1, 'k3', 0, 'v3', '', 3), + (9904, 1, 'k4', 0, 'v4', '', 4), + (9905, 1, 'k5', 0, 'v5', '', 5); diff --git a/doc/images/item-num-limit-enabled.png b/doc/images/item-num-limit-enabled.png new file mode 100644 index 00000000000..efe924fa7d8 Binary files /dev/null and b/doc/images/item-num-limit-enabled.png differ diff --git a/doc/images/item-num-limit.png b/doc/images/item-num-limit.png new file mode 100644 index 00000000000..fc76e712aff Binary files /dev/null and b/doc/images/item-num-limit.png differ diff --git a/docs/en/portal/apollo-user-guide.md b/docs/en/portal/apollo-user-guide.md index d084ab5a0aa..c3890696c0c 100644 --- a/docs/en/portal/apollo-user-guide.md +++ b/docs/en/portal/apollo-user-guide.md @@ -531,6 +531,18 @@ Starting from version 2.4.0, apollo-portal provides the function of checking the ![item-num-limit](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/namespace-num-limit-white.png) +## 6.5 Limitation on the number of configuration items in a single namespace +Starting from version 2.4.0, apollo-portal provides the function of limiting the number of configuration items in a single namespace. This function is disabled by default and needs to be enabled by configuring the system `item.num.limit.enabled`. At the same time, the system parameter `item.num.limit` is provided to dynamically configure the upper limit of the number of items in a single Namespace. + +**Setting method：** +1. Log in to the Apollo Configuration Center interface with a super administrator account +2. Go to the `Administrator Tools - System Parameters - ConfigDB Configuration Management` page and add or modify the `item.num.limit.enabled` configuration item to true/false to enable/disable this function. + + ![item-num-limit-enabled](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit-enabled.png) +3. Go to the `Admin Tools - System Parameters - ConfigDB Configuration Management` page and add or modify the `item.num.limit` configuration item to configure the upper limit of the number of items under a single Namespace. + + ![item-num-limit](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit.png) + # VII. Best practices diff --git a/docs/zh/portal/apollo-user-guide.md b/docs/zh/portal/apollo-user-guide.md index cb38986bb2b..d4616240830 100644 --- a/docs/zh/portal/apollo-user-guide.md +++ b/docs/zh/portal/apollo-user-guide.md @@ -502,6 +502,22 @@ Apollo从1.6.0版本开始增加访问密钥机制，从而只有经过身份验 +## 6.5 单个命名空间下的配置项数量限制 +从2.4.0版本开始，apollo-portal提供了限制单个命名空间下的配置项数量的功能，此功能默认关闭，需要配置系统 `item.num.limit.enabled` 开启，同时提供了系统参数`item.num.limit`来动态配置单个Namespace下的item数量上限值 + +**设置方法：** +1. 用超级管理员账号登录到Apollo配置中心的界面 +2. 进入`管理员工具 - 系统参数 - ConfigDB 配置管理`页面新增或修改`item.num.limit.enabled`配置项为true/false 即可开启/关闭此功能 + + ![item-num-limit-enabled](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit-enabled.png) + +3. 进入`管理员工具 - 系统参数 - ConfigDB 配置管理`页面新增或修改`item.num.limit`配置项来配置单个Namespace下的item数量上限值 + + ![item-num-limit](https://cdn.jsdelivr.net/gh/apolloconfig/apollo@master/doc/images/item-num-limit.png) + + + + # 七、最佳实践 ## 7.1 安全相关