From 1c7dcf8e177b6119fe0542e03e33d9aa1463fe35 Mon Sep 17 00:00:00 2001 From: yangzl Date: Thu, 19 Sep 2024 20:55:41 +0800 Subject: [PATCH] feat: 1. solve the problem of duplicate comments and blank lines 2. reduce the data growth of the item table 3. in the same case, the size of ItemChangeSets can be reduced 4. when revoke configuration, if there is only one comment or blank line, it will not be deleted. --- .../txtresolver/PropertyResolver.java | 64 +++++++++++-------- .../apollo/portal/service/ItemService.java | 12 ++-- .../portal/service/NamespaceService.java | 9 +-- .../txtresolver/PropertyResolverTest.java | 18 +++--- 4 files changed, 59 insertions(+), 44 deletions(-) diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java index 44080d7520d..39772c34763 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java @@ -21,15 +21,20 @@ import com.ctrip.framework.apollo.common.exception.BadRequestException; import com.ctrip.framework.apollo.common.utils.BeanUtils; +import com.ctrip.framework.apollo.core.utils.StringUtils; import com.google.common.base.Strings; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * normal property file resolver. @@ -45,12 +50,21 @@ public class PropertyResolver implements ConfigTextResolver { @Override public ItemChangeSets resolve(long namespaceId, String configText, List baseItems) { - Map oldLineNumMapItem = BeanUtils.mapByKey("lineNum", baseItems); Map oldKeyMapItem = BeanUtils.mapByKey("key", baseItems); - //remove comment and blank item map. oldKeyMapItem.remove(""); + // comment items + List baseCommentItems = new ArrayList<>(); + // blank items + List baseBlankItems = new ArrayList<>(); + if(!CollectionUtils.isEmpty(baseItems)) { + + baseCommentItems = baseItems.stream().filter(itemDTO -> isCommentItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toList()); + + baseBlankItems = baseItems.stream().filter(itemDTO -> isBlankItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toList()); + } + String[] newItems = configText.split(ITEM_SEPARATOR); Set repeatKeys = new HashSet<>(); if (isHasRepeatKey(newItems, repeatKeys)) { @@ -63,17 +77,25 @@ public ItemChangeSets resolve(long namespaceId, String configText, List for (String newItem : newItems) { newItem = newItem.trim(); newLineNumMapItem.put(lineCounter, newItem); - ItemDTO oldItemByLine = oldLineNumMapItem.get(lineCounter); //comment item if (isCommentItem(newItem)) { + ItemDTO oldItemDTO = null; + if(!CollectionUtils.isEmpty(baseCommentItems)) { + oldItemDTO = baseCommentItems.remove(0); + } - handleCommentLine(namespaceId, oldItemByLine, newItem, lineCounter, changeSets); + handleCommentLine(namespaceId, oldItemDTO, newItem, lineCounter, changeSets); //blank item } else if (isBlankItem(newItem)) { - handleBlankLine(namespaceId, oldItemByLine, lineCounter, changeSets); + ItemDTO oldItemDTO = null; + if(!CollectionUtils.isEmpty(baseBlankItems)) { + oldItemDTO = baseBlankItems.remove(0); + } + + handleBlankLine(namespaceId, oldItemDTO, lineCounter, changeSets); //normal item } else { @@ -83,7 +105,7 @@ public ItemChangeSets resolve(long namespaceId, String configText, List lineCounter++; } - deleteCommentAndBlankItem(oldLineNumMapItem, newLineNumMapItem, changeSets); + deleteCommentAndBlankItem(baseCommentItems, baseBlankItems, changeSets); deleteNormalKVItem(oldKeyMapItem, changeSets); return changeSets; @@ -122,16 +144,18 @@ private String[] parseKeyValueFromItem(String item) { } private void handleCommentLine(Long namespaceId, ItemDTO oldItemByLine, String newItem, int lineCounter, ItemChangeSets changeSets) { - String oldComment = oldItemByLine == null ? "" : oldItemByLine.getComment(); - //create comment. implement update comment by delete old comment and create new comment - if (!(isCommentItem(oldItemByLine) && newItem.equals(oldComment))) { + if(null == oldItemByLine ){ changeSets.addCreateItem(buildCommentItem(0L, namespaceId, newItem, lineCounter)); + }else if(!StringUtils.equals(oldItemByLine.getComment(), newItem) || lineCounter != oldItemByLine.getLineNum()) { + changeSets.addUpdateItem(buildCommentItem(oldItemByLine.getId(), namespaceId, newItem, lineCounter)); } } private void handleBlankLine(Long namespaceId, ItemDTO oldItem, int lineCounter, ItemChangeSets changeSets) { - if (!isBlankItem(oldItem)) { + if(null == oldItem ){ changeSets.addCreateItem(buildBlankItem(0L, namespaceId, lineCounter)); + }else if (lineCounter != oldItem.getLineNum()) { + changeSets.addUpdateItem(buildBlankItem(oldItem.getId(), namespaceId, lineCounter)); } } @@ -183,23 +207,11 @@ private void deleteNormalKVItem(Map baseKeyMapItem, ItemChangeS } } - private void deleteCommentAndBlankItem(Map oldLineNumMapItem, - Map newLineNumMapItem, + private void deleteCommentAndBlankItem(List baseCommentItems, + List baseBlankItems, ItemChangeSets changeSets) { - - for (Map.Entry entry : oldLineNumMapItem.entrySet()) { - int lineNum = entry.getKey(); - ItemDTO oldItem = entry.getValue(); - String newItem = newLineNumMapItem.get(lineNum); - - //1. old is blank by now is not - //2.old is comment by now is not exist or modified - //3.old is blank by now is not exist or modified - if ((isBlankItem(oldItem) && !isBlankItem(newItem)) - || (isCommentItem(oldItem) || isBlankItem(oldItem)) && (newItem == null || !newItem.equals(oldItem.getComment()))) { - changeSets.addDeleteItem(oldItem); - } - } + baseCommentItems.forEach(oldItemDTO-> changeSets.addDeleteItem(oldItemDTO)); + baseBlankItems.forEach(oldItemDTO-> changeSets.addDeleteItem(oldItemDTO)); } private ItemDTO buildCommentItem(Long id, Long namespaceId, String comment, int lineNum) { diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java index b1677e18858..2c7aa57c195 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java @@ -50,6 +50,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Service public class ItemService { @@ -216,12 +217,13 @@ public void revokeItem(String appId, Env env, String clusterName, String namespa } List baseItems = itemAPI.findItems(appId, env, clusterName, namespaceName); Map oldKeyMapItem = BeanUtils.mapByKey("key", baseItems); - Map deletedItemDTOs = new HashMap<>(); + //remove comment and blank item map. + oldKeyMapItem.remove(""); //deleted items for comment - findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> { - deletedItemDTOs.put(item.getKey(),item); - }); + Map deletedItemDTOs = findDeletedItems(appId, env, clusterName, namespaceName).stream() + .filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey())) + .collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2)); ItemChangeSets changeSets = new ItemChangeSets(); AtomicInteger lineNum = new AtomicInteger(1); @@ -238,7 +240,7 @@ public void revokeItem(String appId, Env env, String clusterName, String namespa oldKeyMapItem.remove(key); lineNum.set(lineNum.get() + 1); }); - oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(oldKeyMapItem.get(key))); + oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(value)); changeSets.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId()); updateItems(appId, env, clusterName, namespaceName, changeSets); diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java index 43f05e4de94..183a58b891e 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java @@ -309,7 +309,6 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole //latest Release ReleaseDTO latestRelease; Map releaseItems = new HashMap<>(); - Map deletedItemDTOs = new HashMap<>(); latestRelease = releaseService.loadLatestRelease(appId, env, clusterName, namespaceName); if (latestRelease != null) { releaseItems = GSON.fromJson(latestRelease.getConfigurations(), GsonType.CONFIG); @@ -333,9 +332,9 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole if (includeDeletedItems) { //deleted items - itemService.findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> { - deletedItemDTOs.put(item.getKey(), item); - }); + Map deletedItemDTOs = itemService.findDeletedItems(appId, env, clusterName, namespaceName).stream() + .filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey())) + .collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2)); List deletedItems = parseDeletedItems(items, releaseItems, deletedItemDTOs); itemBOs.addAll(deletedItems); @@ -385,6 +384,8 @@ private void fillAppNamespaceProperties(NamespaceBO namespace) { private List parseDeletedItems(List newItems, Map releaseItems, Map deletedItemDTOs) { Map newItemMap = BeanUtils.mapByKey("key", newItems); + //remove comment and blank item map. + newItemMap.remove(""); List deletedItems = new LinkedList<>(); for (Map.Entry entry : releaseItems.entrySet()) { diff --git a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java index 4c8664f7b47..ed18a70c448 100644 --- a/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java +++ b/apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java @@ -91,9 +91,9 @@ public void testDeleteItem() { @Test public void testDeleteCommentItem() { ItemChangeSets changeSets = resolver.resolve(1, "a=b\n\nb=c", mockBaseItemWith2Key1Comment1Blank()); - Assert.assertEquals(2, changeSets.getDeleteItems().size()); - Assert.assertEquals(2, changeSets.getUpdateItems().size()); - Assert.assertEquals(1, changeSets.getCreateItems().size()); + Assert.assertEquals(1, changeSets.getDeleteItems().size()); + Assert.assertEquals(3, changeSets.getUpdateItems().size()); + Assert.assertEquals(0, changeSets.getCreateItems().size()); } @Test @@ -120,17 +120,17 @@ public void testUpdateCommentItem() { + "a=b\n" +"\n" + "b=c", mockBaseItemWith2Key1Comment1Blank()); - Assert.assertEquals(1, changeSets.getDeleteItems().size()); - Assert.assertEquals(0, changeSets.getUpdateItems().size()); - Assert.assertEquals(1, changeSets.getCreateItems().size()); + Assert.assertEquals(0, changeSets.getDeleteItems().size()); + Assert.assertEquals(1, changeSets.getUpdateItems().size()); + Assert.assertEquals(0, changeSets.getCreateItems().size()); } @Test public void testAllSituation(){ ItemChangeSets changeSets = resolver.resolve(1, "#ww\nd=e\nb=c\na=b\n\nq=w\n#eee", mockBaseItemWith2Key1Comment1Blank()); - Assert.assertEquals(2, changeSets.getDeleteItems().size()); - Assert.assertEquals(2, changeSets.getUpdateItems().size()); - Assert.assertEquals(5, changeSets.getCreateItems().size()); + Assert.assertEquals(0, changeSets.getDeleteItems().size()); + Assert.assertEquals(4, changeSets.getUpdateItems().size()); + Assert.assertEquals(3, changeSets.getCreateItems().size()); } /**