Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

solve the problem of duplicate comments and blank lines #5232

Merged
merged 9 commits into from
Oct 1, 2024
Merged

solve the problem of duplicate comments and blank lines #5232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1c7dcf8
feat:
youngzil Sep 19, 2024
444ce9b
Merge branch 'master' into feature/comment_and_blank_repeat
youngzil Sep 29, 2024
859bbdb
fix：
youngzil Sep 29, 2024
904ab87
fix：code optimization
youngzil Sep 29, 2024
09b4601
feat:
youngzil Sep 19, 2024
fdc5e60
fix：
youngzil Sep 29, 2024
48da7db
fix：code optimization
youngzil Sep 29, 2024
1c024df
Merge remote-tracking branch 'youngzil/feature/comment_and_blank_repe…
youngzil Sep 29, 2024
84180a2
Merge branch 'master' into feature/comment_and_blank_repeat
youngzil Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ Apollo 2.4.0
* [Update the server config link in system info page](https://github.com/apolloconfig/apollo/pull/5204)
* [Feature support portal restTemplate Client connection pool config](https://github.com/apolloconfig/apollo/pull/5200)
* [Feature added the ability for administrators to globally search for Value](https://github.com/apolloconfig/apollo/pull/5182)
* [Fix: Resolve issues with duplicate comments and blank lines in configuration management](https://github.com/apolloconfig/apollo/pull/5232)
* [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240)

------------------
All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
78 changes: 45 additions & 33 deletions ...c/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,20 @@
import com.ctrip.framework.apollo.common.exception.BadRequestException;
import com.ctrip.framework.apollo.common.utils.BeanUtils;

import com.ctrip.framework.apollo.core.utils.StringUtils;
import com.google.common.base.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import javax.validation.constraints.NotNull;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
* normal property file resolver.
Expand All @@ -45,12 +50,21 @@ public class PropertyResolver implements ConfigTextResolver {
@Override
public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO> baseItems) {

Map<Integer, ItemDTO> oldLineNumMapItem = BeanUtils.mapByKey("lineNum", baseItems);
Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);

//remove comment and blank item map.
oldKeyMapItem.remove("");

// comment items
List<ItemDTO> baseCommentItems = new LinkedList<>();
// blank items
List<ItemDTO> baseBlankItems = new LinkedList<>();
if (!CollectionUtils.isEmpty(baseItems)) {

baseCommentItems = baseItems.stream().filter(itemDTO -> isCommentItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toCollection(LinkedList::new));

baseBlankItems = baseItems.stream().filter(itemDTO -> isBlankItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toCollection(LinkedList::new));
}

String[] newItems = configText.split(ITEM_SEPARATOR);
Set<String> repeatKeys = new HashSet<>();
if (isHasRepeatKey(newItems, repeatKeys)) {
Expand All @@ -63,17 +77,25 @@ public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO>
for (String newItem : newItems) {
newItem = newItem.trim();
newLineNumMapItem.put(lineCounter, newItem);
ItemDTO oldItemByLine = oldLineNumMapItem.get(lineCounter);

//comment item
if (isCommentItem(newItem)) {
ItemDTO oldItemDTO = null;
if (!CollectionUtils.isEmpty(baseCommentItems)) {
oldItemDTO = baseCommentItems.remove(0);
}

handleCommentLine(namespaceId, oldItemByLine, newItem, lineCounter, changeSets);
handleCommentLine(namespaceId, oldItemDTO, newItem, lineCounter, changeSets);

//blank item
} else if (isBlankItem(newItem)) {

handleBlankLine(namespaceId, oldItemByLine, lineCounter, changeSets);
ItemDTO oldItemDTO = null;
if (!CollectionUtils.isEmpty(baseBlankItems)) {
oldItemDTO = baseBlankItems.remove(0);
}

handleBlankLine(namespaceId, oldItemDTO, lineCounter, changeSets);

//normal item
} else {
Expand All @@ -83,7 +105,7 @@ public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO>
lineCounter++;
}

deleteCommentAndBlankItem(oldLineNumMapItem, newLineNumMapItem, changeSets);
deleteCommentAndBlankItem(baseCommentItems, baseBlankItems, changeSets);
deleteNormalKVItem(oldKeyMapItem, changeSets);

return changeSets;
Expand All @@ -97,7 +119,7 @@ private boolean isHasRepeatKey(String[] newItems, @NotNull Set<String> repeatKey
String[] kv = parseKeyValueFromItem(item);
if (kv != null) {
String key = kv[0].toLowerCase();
if(!keys.add(key)){
if (!keys.add(key)) {
repeatKeys.add(key);
}
} else {
Expand All @@ -122,16 +144,18 @@ private String[] parseKeyValueFromItem(String item) {
}

private void handleCommentLine(Long namespaceId, ItemDTO oldItemByLine, String newItem, int lineCounter, ItemChangeSets changeSets) {
String oldComment = oldItemByLine == null ? "" : oldItemByLine.getComment();
//create comment. implement update comment by delete old comment and create new comment
if (!(isCommentItem(oldItemByLine) && newItem.equals(oldComment))) {
if (null == oldItemByLine) {
changeSets.addCreateItem(buildCommentItem(0L, namespaceId, newItem, lineCounter));
} else if (!StringUtils.equals(oldItemByLine.getComment(), newItem) || lineCounter != oldItemByLine.getLineNum()) {
changeSets.addUpdateItem(buildCommentItem(oldItemByLine.getId(), namespaceId, newItem, lineCounter));
}
}

private void handleBlankLine(Long namespaceId, ItemDTO oldItem, int lineCounter, ItemChangeSets changeSets) {
if (!isBlankItem(oldItem)) {
if (null == oldItem) {
changeSets.addCreateItem(buildBlankItem(0L, namespaceId, lineCounter));
} else if (lineCounter != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildBlankItem(oldItem.getId(), namespaceId, lineCounter));
}
}

Expand All @@ -149,12 +173,12 @@ private void handleNormalLine(Long namespaceId, Map<String, ItemDTO> keyMapOldIt

ItemDTO oldItem = keyMapOldItem.get(newKey);

if (oldItem == null) {//new item
//new item
if (oldItem == null) {
changeSets.addCreateItem(buildNormalItem(0L, namespaceId, newKey, newValue, "", lineCounter));
} else if (!newValue.equals(oldItem.getValue()) || lineCounter != oldItem.getLineNum()) {//update item
changeSets.addUpdateItem(
buildNormalItem(oldItem.getId(), namespaceId, newKey, newValue, oldItem.getComment(),
lineCounter));
//update item
} else if (!StringUtils.equals(newValue, oldItem.getValue()) || lineCounter != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildNormalItem(oldItem.getId(), namespaceId, newKey, newValue, oldItem.getComment(), lineCounter));
}
keyMapOldItem.remove(newKey);
}
Expand All @@ -173,7 +197,7 @@ private boolean isBlankItem(ItemDTO item) {
}

private boolean isBlankItem(String line) {
return Strings.nullToEmpty(line).trim().isEmpty();
return Strings.nullToEmpty(line).trim().isEmpty();
}

private void deleteNormalKVItem(Map<String, ItemDTO> baseKeyMapItem, ItemChangeSets changeSets) {
Expand All @@ -183,23 +207,11 @@ private void deleteNormalKVItem(Map<String, ItemDTO> baseKeyMapItem, ItemChangeS
}
}

private void deleteCommentAndBlankItem(Map<Integer, ItemDTO> oldLineNumMapItem,
Map<Integer, String> newLineNumMapItem,
private void deleteCommentAndBlankItem(List<ItemDTO> baseCommentItems,
List<ItemDTO> baseBlankItems,
ItemChangeSets changeSets) {

for (Map.Entry<Integer, ItemDTO> entry : oldLineNumMapItem.entrySet()) {
int lineNum = entry.getKey();
ItemDTO oldItem = entry.getValue();
String newItem = newLineNumMapItem.get(lineNum);

//1. old is blank by now is not
//2.old is comment by now is not exist or modified
//3.old is blank by now is not exist or modified
if ((isBlankItem(oldItem) && !isBlankItem(newItem))
|| (isCommentItem(oldItem) || isBlankItem(oldItem)) && (newItem == null || !newItem.equals(oldItem.getComment()))) {
changeSets.addDeleteItem(oldItem);
}
}
baseCommentItems.forEach(oldItemDTO -> changeSets.addDeleteItem(oldItemDTO));
baseBlankItems.forEach(oldItemDTO -> changeSets.addDeleteItem(oldItemDTO));
}

private ItemDTO buildCommentItem(Long id, Long namespaceId, String comment, int lineNum) {
Expand Down
21 changes: 11 additions & 10 deletions apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@Service
public class ItemService {
Expand Down Expand Up @@ -216,29 +217,29 @@ public void revokeItem(String appId, Env env, String clusterName, String namespa
}
List<ItemDTO> baseItems = itemAPI.findItems(appId, env, clusterName, namespaceName);
Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);
Map<String, ItemDTO> deletedItemDTOs = new HashMap<>();
//remove comment and blank item map.
oldKeyMapItem.remove("");
Comment on lines +220 to +221
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Consider filtering out blank keys during map creation

Instead of removing entries with blank keys after creating oldKeyMapItem, consider filtering out items with blank keys when constructing the map. This can improve code clarity and prevent unnecessary entries from being processed.

You can modify the map creation as follows:

-import java.util.HashMap;
+import java.util.HashMap;
+import java.util.stream.Collectors;

 Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);
-//remove comment and blank item map.
-oldKeyMapItem.remove("");
+oldKeyMapItem = oldKeyMapItem.entrySet().stream()
+    .filter(entry -> !StringUtils.isEmpty(entry.getKey()))
+    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

Committable suggestion was skipped due to low confidence.


//deleted items for comment
findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> {
deletedItemDTOs.put(item.getKey(),item);
});
Map<String, ItemDTO> deletedItemDTOs = findDeletedItems(appId, env, clusterName, namespaceName).stream()
.filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey()))
.collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2));

ItemChangeSets changeSets = new ItemChangeSets();
AtomicInteger lineNum = new AtomicInteger(1);
releaseItemDTOs.forEach((key,value) -> {
ItemDTO oldItem = oldKeyMapItem.get(key);
if (oldItem == null) {
ItemDTO deletedItemDto = deletedItemDTOs.computeIfAbsent(key, k -> new ItemDTO());
changeSets.addCreateItem(buildNormalItem(0L, namespaceId,key,value,deletedItemDto.getComment(),lineNum.get()));
} else if (!oldItem.getValue().equals(value) || lineNum.get() != oldItem
.getLineNum()) {
changeSets.addUpdateItem(buildNormalItem(oldItem.getId(), namespaceId, key,
value, oldItem.getComment(), lineNum.get()));
int newLineNum = 0 == deletedItemDto.getLineNum() ? lineNum.get() : deletedItemDto.getLineNum();
changeSets.addCreateItem(buildNormalItem(0L, namespaceId, key, value, deletedItemDto.getComment(), newLineNum));
} else if (!StringUtils.equals(oldItem.getValue(), value) || lineNum.get() != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildNormalItem(oldItem.getId(), namespaceId, key, value, oldItem.getComment(), oldItem.getLineNum()));
}
oldKeyMapItem.remove(key);
lineNum.set(lineNum.get() + 1);
});
oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(oldKeyMapItem.get(key)));
oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(value));
changeSets.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId());

updateItems(appId, env, clusterName, namespaceName, changeSets);
Expand Down
9 changes: 5 additions & 4 deletions apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,6 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole
//latest Release
ReleaseDTO latestRelease;
Map<String, String> releaseItems = new HashMap<>();
Map<String, ItemDTO> deletedItemDTOs = new HashMap<>();
latestRelease = releaseService.loadLatestRelease(appId, env, clusterName, namespaceName);
if (latestRelease != null) {
releaseItems = GSON.fromJson(latestRelease.getConfigurations(), GsonType.CONFIG);
Expand All @@ -333,9 +332,9 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole

if (includeDeletedItems) {
//deleted items
itemService.findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> {
deletedItemDTOs.put(item.getKey(), item);
});
Map<String, ItemDTO> deletedItemDTOs = itemService.findDeletedItems(appId, env, clusterName, namespaceName).stream()
.filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey()))
.collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2));

List<ItemBO> deletedItems = parseDeletedItems(items, releaseItems, deletedItemDTOs);
itemBOs.addAll(deletedItems);
Expand Down Expand Up @@ -385,6 +384,8 @@ private void fillAppNamespaceProperties(NamespaceBO namespace) {

private List<ItemBO> parseDeletedItems(List<ItemDTO> newItems, Map<String, String> releaseItems, Map<String, ItemDTO> deletedItemDTOs) {
Map<String, ItemDTO> newItemMap = BeanUtils.mapByKey("key", newItems);
//remove comment and blank item map.
newItemMap.remove("");

List<ItemBO> deletedItems = new LinkedList<>();
for (Map.Entry<String, String> entry : releaseItems.entrySet()) {
Expand Down
18 changes: 9 additions & 9 deletions ...st/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ public void testDeleteItem() {
@Test
public void testDeleteCommentItem() {
ItemChangeSets changeSets = resolver.resolve(1, "a=b\n\nb=c", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(2, changeSets.getDeleteItems().size());
Assert.assertEquals(2, changeSets.getUpdateItems().size());
Assert.assertEquals(1, changeSets.getCreateItems().size());
Assert.assertEquals(1, changeSets.getDeleteItems().size());
Assert.assertEquals(3, changeSets.getUpdateItems().size());
Assert.assertEquals(0, changeSets.getCreateItems().size());
}

@Test
Expand All @@ -120,17 +120,17 @@ public void testUpdateCommentItem() {
+ "a=b\n"
+"\n"
+ "b=c", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(1, changeSets.getDeleteItems().size());
Assert.assertEquals(0, changeSets.getUpdateItems().size());
Assert.assertEquals(1, changeSets.getCreateItems().size());
Assert.assertEquals(0, changeSets.getDeleteItems().size());
Assert.assertEquals(1, changeSets.getUpdateItems().size());
Assert.assertEquals(0, changeSets.getCreateItems().size());
}

@Test
public void testAllSituation(){
ItemChangeSets changeSets = resolver.resolve(1, "#ww\nd=e\nb=c\na=b\n\nq=w\n#eee", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(2, changeSets.getDeleteItems().size());
Assert.assertEquals(2, changeSets.getUpdateItems().size());
Assert.assertEquals(5, changeSets.getCreateItems().size());
Assert.assertEquals(0, changeSets.getDeleteItems().size());
Assert.assertEquals(4, changeSets.getUpdateItems().size());
Assert.assertEquals(3, changeSets.getCreateItems().size());
}

/**
Expand Down
Loading