Skip to content

Commit

Permalink
feat:
Browse files Browse the repository at this point in the history 
1. solve the problem of duplicate comments and blank lines
2. reduce the data growth of the item table
3. in the same case, the size of ItemChangeSets can be reduced
4. when revoke configuration, if there is only one comment or blank line, it will not be deleted.
  • Loading branch information
@youngzil
youngzil committed Sep 19, 2024
1 parent 31e6486 commit 1c7dcf8
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 44 deletions.
64 changes: 38 additions & 26 deletions ...c/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,20 @@
import com.ctrip.framework.apollo.common.exception.BadRequestException;
import com.ctrip.framework.apollo.common.utils.BeanUtils;

import com.ctrip.framework.apollo.core.utils.StringUtils;
import com.google.common.base.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import javax.validation.constraints.NotNull;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
* normal property file resolver.
Expand All @@ -45,12 +50,21 @@ public class PropertyResolver implements ConfigTextResolver {
@Override
public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO> baseItems) {

Map<Integer, ItemDTO> oldLineNumMapItem = BeanUtils.mapByKey("lineNum", baseItems);
Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);

//remove comment and blank item map.
oldKeyMapItem.remove("");

// comment items
List<ItemDTO> baseCommentItems = new ArrayList<>();
// blank items
List<ItemDTO> baseBlankItems = new ArrayList<>();
if(!CollectionUtils.isEmpty(baseItems)) {

baseCommentItems = baseItems.stream().filter(itemDTO -> isCommentItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toList());

baseBlankItems = baseItems.stream().filter(itemDTO -> isBlankItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toList());
}

String[] newItems = configText.split(ITEM_SEPARATOR);
Set<String> repeatKeys = new HashSet<>();
if (isHasRepeatKey(newItems, repeatKeys)) {
Expand All @@ -63,17 +77,25 @@ public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO>
for (String newItem : newItems) {
newItem = newItem.trim();
newLineNumMapItem.put(lineCounter, newItem);
ItemDTO oldItemByLine = oldLineNumMapItem.get(lineCounter);

//comment item
if (isCommentItem(newItem)) {
ItemDTO oldItemDTO = null;
if(!CollectionUtils.isEmpty(baseCommentItems)) {
oldItemDTO = baseCommentItems.remove(0);
}

handleCommentLine(namespaceId, oldItemByLine, newItem, lineCounter, changeSets);
handleCommentLine(namespaceId, oldItemDTO, newItem, lineCounter, changeSets);

//blank item
} else if (isBlankItem(newItem)) {

handleBlankLine(namespaceId, oldItemByLine, lineCounter, changeSets);
ItemDTO oldItemDTO = null;
if(!CollectionUtils.isEmpty(baseBlankItems)) {
oldItemDTO = baseBlankItems.remove(0);
}

handleBlankLine(namespaceId, oldItemDTO, lineCounter, changeSets);

//normal item
} else {
Expand All @@ -83,7 +105,7 @@ public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO>
lineCounter++;
}

deleteCommentAndBlankItem(oldLineNumMapItem, newLineNumMapItem, changeSets);
deleteCommentAndBlankItem(baseCommentItems, baseBlankItems, changeSets);
deleteNormalKVItem(oldKeyMapItem, changeSets);

return changeSets;
Expand Down Expand Up @@ -122,16 +144,18 @@ private String[] parseKeyValueFromItem(String item) {
}

private void handleCommentLine(Long namespaceId, ItemDTO oldItemByLine, String newItem, int lineCounter, ItemChangeSets changeSets) {
String oldComment = oldItemByLine == null ? "" : oldItemByLine.getComment();
//create comment. implement update comment by delete old comment and create new comment
if (!(isCommentItem(oldItemByLine) && newItem.equals(oldComment))) {
if(null == oldItemByLine ){
changeSets.addCreateItem(buildCommentItem(0L, namespaceId, newItem, lineCounter));
}else if(!StringUtils.equals(oldItemByLine.getComment(), newItem) || lineCounter != oldItemByLine.getLineNum()) {
changeSets.addUpdateItem(buildCommentItem(oldItemByLine.getId(), namespaceId, newItem, lineCounter));
}
}

private void handleBlankLine(Long namespaceId, ItemDTO oldItem, int lineCounter, ItemChangeSets changeSets) {
if (!isBlankItem(oldItem)) {
if(null == oldItem ){
changeSets.addCreateItem(buildBlankItem(0L, namespaceId, lineCounter));
}else if (lineCounter != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildBlankItem(oldItem.getId(), namespaceId, lineCounter));
}
}

Expand Down Expand Up @@ -183,23 +207,11 @@ private void deleteNormalKVItem(Map<String, ItemDTO> baseKeyMapItem, ItemChangeS
}
}

private void deleteCommentAndBlankItem(Map<Integer, ItemDTO> oldLineNumMapItem,
Map<Integer, String> newLineNumMapItem,
private void deleteCommentAndBlankItem(List<ItemDTO> baseCommentItems,
List<ItemDTO> baseBlankItems,
ItemChangeSets changeSets) {

for (Map.Entry<Integer, ItemDTO> entry : oldLineNumMapItem.entrySet()) {
int lineNum = entry.getKey();
ItemDTO oldItem = entry.getValue();
String newItem = newLineNumMapItem.get(lineNum);

//1. old is blank by now is not
//2.old is comment by now is not exist or modified
//3.old is blank by now is not exist or modified
if ((isBlankItem(oldItem) && !isBlankItem(newItem))
|| (isCommentItem(oldItem) || isBlankItem(oldItem)) && (newItem == null || !newItem.equals(oldItem.getComment()))) {
changeSets.addDeleteItem(oldItem);
}
}
baseCommentItems.forEach(oldItemDTO-> changeSets.addDeleteItem(oldItemDTO));
baseBlankItems.forEach(oldItemDTO-> changeSets.addDeleteItem(oldItemDTO));
}

private ItemDTO buildCommentItem(Long id, Long namespaceId, String comment, int lineNum) {
Expand Down
12 changes: 7 additions & 5 deletions apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@Service
public class ItemService {
Expand Down Expand Up @@ -216,12 +217,13 @@ public void revokeItem(String appId, Env env, String clusterName, String namespa
}
List<ItemDTO> baseItems = itemAPI.findItems(appId, env, clusterName, namespaceName);
Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);
Map<String, ItemDTO> deletedItemDTOs = new HashMap<>();
//remove comment and blank item map.
oldKeyMapItem.remove("");

//deleted items for comment
findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> {
deletedItemDTOs.put(item.getKey(),item);
});
Map<String, ItemDTO> deletedItemDTOs = findDeletedItems(appId, env, clusterName, namespaceName).stream()
.filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey()))
.collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2));

ItemChangeSets changeSets = new ItemChangeSets();
AtomicInteger lineNum = new AtomicInteger(1);
Expand All @@ -238,7 +240,7 @@ public void revokeItem(String appId, Env env, String clusterName, String namespa
oldKeyMapItem.remove(key);
lineNum.set(lineNum.get() + 1);
});
oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(oldKeyMapItem.get(key)));
oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(value));
changeSets.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId());

updateItems(appId, env, clusterName, namespaceName, changeSets);
Expand Down
9 changes: 5 additions & 4 deletions apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,6 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole
//latest Release
ReleaseDTO latestRelease;
Map<String, String> releaseItems = new HashMap<>();
Map<String, ItemDTO> deletedItemDTOs = new HashMap<>();
latestRelease = releaseService.loadLatestRelease(appId, env, clusterName, namespaceName);
if (latestRelease != null) {
releaseItems = GSON.fromJson(latestRelease.getConfigurations(), GsonType.CONFIG);
Expand All @@ -333,9 +332,9 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole

if (includeDeletedItems) {
//deleted items
itemService.findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> {
deletedItemDTOs.put(item.getKey(), item);
});
Map<String, ItemDTO> deletedItemDTOs = itemService.findDeletedItems(appId, env, clusterName, namespaceName).stream()
.filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey()))
.collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2));

List<ItemBO> deletedItems = parseDeletedItems(items, releaseItems, deletedItemDTOs);
itemBOs.addAll(deletedItems);
Expand Down Expand Up @@ -385,6 +384,8 @@ private void fillAppNamespaceProperties(NamespaceBO namespace) {

private List<ItemBO> parseDeletedItems(List<ItemDTO> newItems, Map<String, String> releaseItems, Map<String, ItemDTO> deletedItemDTOs) {
Map<String, ItemDTO> newItemMap = BeanUtils.mapByKey("key", newItems);
//remove comment and blank item map.
newItemMap.remove("");

List<ItemBO> deletedItems = new LinkedList<>();
for (Map.Entry<String, String> entry : releaseItems.entrySet()) {
Expand Down
18 changes: 9 additions & 9 deletions ...st/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ public void testDeleteItem() {
@Test
public void testDeleteCommentItem() {
ItemChangeSets changeSets = resolver.resolve(1, "a=b\n\nb=c", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(2, changeSets.getDeleteItems().size());
Assert.assertEquals(2, changeSets.getUpdateItems().size());
Assert.assertEquals(1, changeSets.getCreateItems().size());
Assert.assertEquals(1, changeSets.getDeleteItems().size());
Assert.assertEquals(3, changeSets.getUpdateItems().size());
Assert.assertEquals(0, changeSets.getCreateItems().size());
}

@Test
Expand All @@ -120,17 +120,17 @@ public void testUpdateCommentItem() {
+ "a=b\n"
+"\n"
+ "b=c", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(1, changeSets.getDeleteItems().size());
Assert.assertEquals(0, changeSets.getUpdateItems().size());
Assert.assertEquals(1, changeSets.getCreateItems().size());
Assert.assertEquals(0, changeSets.getDeleteItems().size());
Assert.assertEquals(1, changeSets.getUpdateItems().size());
Assert.assertEquals(0, changeSets.getCreateItems().size());
}

@Test
public void testAllSituation(){
ItemChangeSets changeSets = resolver.resolve(1, "#ww\nd=e\nb=c\na=b\n\nq=w\n#eee", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(2, changeSets.getDeleteItems().size());
Assert.assertEquals(2, changeSets.getUpdateItems().size());
Assert.assertEquals(5, changeSets.getCreateItems().size());
Assert.assertEquals(0, changeSets.getDeleteItems().size());
Assert.assertEquals(4, changeSets.getUpdateItems().size());
Assert.assertEquals(3, changeSets.getCreateItems().size());
}

/**
Expand Down

0 comments on commit 1c7dcf8

Please sign in to comment.