From 4278b483caeefec18379a5efcc20f81074e0bdaf Mon Sep 17 00:00:00 2001 From: Gagharv Date: Mon, 6 Jul 2020 10:08:32 +0800 Subject: [PATCH] Resolve the issues of codestyle of nacos-config module for phase2 (#3253) * [ISSUE#3249]Resolve the part issues of codestyle for nacos-config module(package:service/dump) Change-Id: I273493054e3d7f5f26d73d65fa8360266a93cbfc * [ISSUE#3249]Resolve the part issues of codestyle for nacos-config module(package:service/merge) Change-Id: I8cdf601c2cb5f177a920c0f690cfbace8108c4cc --- .../service/dump/DumpConfigHandler.java | 9 + .../server/service/dump/DumpService.java | 72 ++-- .../config/server/service/dump/DumpTask.java | 388 ------------------ .../service/dump/EmbeddedDumpService.java | 9 +- .../service/dump/ExternalDumpService.java | 4 +- .../dump/processor/DumpAllBetaProcessor.java | 75 ++++ .../dump/processor/DumpAllProcessor.java | 94 +++++ .../dump/processor/DumpAllTagProcessor.java | 74 ++++ .../dump/processor/DumpChangeProcessor.java | 104 +++++ .../service/dump/processor/DumpProcessor.java | 93 +++++ .../service/dump/task/DumpAllBetaTask.java | 34 ++ .../service/dump/task/DumpAllTagTask.java | 34 ++ .../server/service/dump/task/DumpAllTask.java | 34 ++ .../service/dump/task/DumpChangeTask.java | 34 ++ .../server/service/dump/task/DumpTask.java | 94 +++++ .../server/service/merge/MergeDataTask.java | 4 +- .../service/merge/MergeDatumService.java | 39 +- .../service/merge/MergeTaskProcessor.java | 32 +- 18 files changed, 765 insertions(+), 462 deletions(-) delete mode 100755 config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpTask.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllBetaProcessor.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllProcessor.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllTagProcessor.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpChangeProcessor.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpProcessor.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllBetaTask.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTagTask.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTask.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpChangeTask.java create mode 100755 config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpTask.java diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpConfigHandler.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpConfigHandler.java index eb66f290680..758b22150b7 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpConfigHandler.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpConfigHandler.java @@ -27,10 +27,19 @@ import com.alibaba.nacos.core.notify.listener.Subscribe; /** + * Dump config subscriber. + * * @author liaochuntao */ public class DumpConfigHandler implements Subscribe { + /** + * trigger config dump event. + * + * @param event {@link ConfigDumpEvent} + * @return {@code true} if the config dump task success , else + * {@code false} + */ public static boolean configDump(ConfigDumpEvent event) { final String dataId = event.getDataId(); final String group = event.getGroup(); diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java index 1dc0ca42dcd..6643c2aa77d 100755 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java @@ -29,6 +29,16 @@ import com.alibaba.nacos.config.server.model.Page; import com.alibaba.nacos.config.server.service.ConfigCacheService; import com.alibaba.nacos.config.server.service.datasource.DynamicDataSource; +import com.alibaba.nacos.config.server.service.dump.processor.DumpAllBetaProcessor; +import com.alibaba.nacos.config.server.service.dump.processor.DumpAllProcessor; +import com.alibaba.nacos.config.server.service.dump.processor.DumpAllTagProcessor; +import com.alibaba.nacos.config.server.service.dump.processor.DumpChangeProcessor; +import com.alibaba.nacos.config.server.service.dump.processor.DumpProcessor; +import com.alibaba.nacos.config.server.service.dump.task.DumpAllBetaTask; +import com.alibaba.nacos.config.server.service.dump.task.DumpAllTagTask; +import com.alibaba.nacos.config.server.service.dump.task.DumpAllTask; +import com.alibaba.nacos.config.server.service.dump.task.DumpChangeTask; +import com.alibaba.nacos.config.server.service.dump.task.DumpTask; import com.alibaba.nacos.config.server.service.merge.MergeTaskProcessor; import com.alibaba.nacos.config.server.service.repository.PersistService; import com.alibaba.nacos.config.server.utils.ConfigExecutor; @@ -60,7 +70,7 @@ import static com.alibaba.nacos.config.server.utils.LogUtil.fatalLog; /** - * Dump data service + * Dump data service. * * @author Nacos */ @@ -81,7 +91,7 @@ public abstract class DumpService { /** * Here you inject the dependent objects constructively, ensuring that some of the dependent functionality is - * initialized ahead of time + * initialized ahead of time. * * @param persistService {@link PersistService} * @param memberManager {@link ServerMemberManager} @@ -115,9 +125,9 @@ public ServerMemberManager getMemberManager() { } /** - * initialize + * initialize. * - * @throws Throwable + * @throws Throwable throws Exception when actually operate. */ protected abstract void init() throws Throwable; @@ -134,7 +144,7 @@ protected void dumpOperate(DumpProcessor processor, DumpAllProcessor dumpAllProc Runnable dumpAllTag = () -> dumpAllTaskMgr.addTask(DumpAllTagTask.TASK_ID, new DumpAllTagTask()); Runnable clearConfigHistory = () -> { - log.warn("clearConfigHistory start"); + LOGGER.warn("clearConfigHistory start"); if (canExecute()) { try { Timestamp startTime = getBeforeStamp(TimeUtils.getCurrentTime(), 24 * getRetentionDays()); @@ -142,16 +152,16 @@ protected void dumpOperate(DumpProcessor processor, DumpAllProcessor dumpAllProc if (totalCount > 0) { int pageSize = 1000; int removeTime = (totalCount + pageSize - 1) / pageSize; - log.warn("clearConfigHistory, getBeforeStamp:{}, totalCount:{}, pageSize:{}, removeTime:{}", + LOGGER.warn("clearConfigHistory, getBeforeStamp:{}, totalCount:{}, pageSize:{}, removeTime:{}", startTime, totalCount, pageSize, removeTime); while (removeTime > 0) { - // 分页删除，以免批量太大报错 + // delete paging to avoid reporting errors in batches persistService.removeConfigHistory(startTime, pageSize); removeTime--; } } } catch (Throwable e) { - log.error("clearConfigHistory error : {}", e.toString()); + LOGGER.error("clearConfigHistory error : {}", e.toString()); } } }; @@ -159,13 +169,13 @@ protected void dumpOperate(DumpProcessor processor, DumpAllProcessor dumpAllProc try { dumpConfigInfo(dumpAllProcessor); - // 更新beta缓存 + // update Beta cache LogUtil.defaultLog.info("start clear all config-info-beta."); DiskUtil.clearAllBeta(); if (persistService.isExistTable(BETA_TABLE_NAME)) { dumpAllBetaProcessor.process(DumpAllBetaTask.TASK_ID, new DumpAllBetaTask()); } - // 更新Tag缓存 + // update Tag cache LogUtil.defaultLog.info("start clear all config-info-tag."); DiskUtil.clearAllTag(); if (persistService.isExistTable(TAG_TABLE_NAME)) { @@ -181,7 +191,7 @@ protected void dumpOperate(DumpProcessor processor, DumpAllProcessor dumpAllProc MergeAllDataWorker work = new MergeAllDataWorker(list); work.start(); } - log.info("server start, schedule merge end."); + LOGGER.info("server start, schedule merge end."); } } catch (Exception e) { LogUtil.fatalLog @@ -284,13 +294,8 @@ private void dumpConfigInfo(DumpAllProcessor dumpAllProcessor) throws IOExceptio private Timestamp getBeforeStamp(Timestamp date, int step) { Calendar cal = Calendar.getInstance(); - /** - * date 换成已经已知的Date对象 - */ cal.setTime(date); - /** - * before 6 hour - */ + // before 6 hour cal.add(Calendar.HOUR_OF_DAY, -step); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return Timestamp.valueOf(format.format(cal.getTime())); @@ -393,13 +398,13 @@ public void run() { .findConfigInfoAggrByPage(dataId, group, tenant, pageNo, PAGE_SIZE); if (page != null) { datumList.addAll(page.getPageItems()); - log.info("[merge-query] {}, {}, size/total={}/{}", dataId, group, datumList.size(), + LOGGER.info("[merge-query] {}, {}, size/total={}/{}", dataId, group, datumList.size(), rowCount); } } final Timestamp time = TimeUtils.getCurrentTime(); - // 聚合 + // merge if (datumList.size() > 0) { ConfigInfo cf = MergeTaskProcessor.merge(dataId, group, tenant, datumList); String aggrContent = cf.getContent(); @@ -408,44 +413,43 @@ public void run() { if (!StringUtils.equals(localContentMD5, aggrConetentMD5)) { persistService.insertOrUpdate(null, null, cf, time, null, false); - log.info("[merge-ok] {}, {}, size={}, length={}, md5={}, content={}", dataId, group, + LOGGER.info("[merge-ok] {}, {}, size={}, length={}, md5={}, content={}", dataId, group, datumList.size(), cf.getContent().length(), cf.getMd5(), ContentUtils.truncateContent(cf.getContent())); } - } - // 删除 - else { + } else { + // remove config info persistService.removeConfigInfo(dataId, group, tenant, InetUtils.getSelfIp(), null); - log.warn("[merge-delete] delete config info because no datum. dataId=" + dataId + ", groupId=" + LOGGER.warn("[merge-delete] delete config info because no datum. dataId=" + dataId + ", groupId=" + group); } } catch (Throwable e) { - log.info("[merge-error] " + dataId + ", " + group + ", " + e.toString(), e); + LOGGER.info("[merge-error] " + dataId + ", " + group + ", " + e.toString(), e); } FINISHED.incrementAndGet(); if (FINISHED.get() % 100 == 0) { - log.info("[all-merge-dump] {} / {}", FINISHED.get(), total); + LOGGER.info("[all-merge-dump] {} / {}", FINISHED.get(), total); } } - log.info("[all-merge-dump] {} / {}", FINISHED.get(), total); + LOGGER.info("[all-merge-dump] {} / {}", FINISHED.get(), total); } } /** - * Used to determine whether the aggregation task, configuration history cleanup task can be performed + * Used to determine whether the aggregation task, configuration history cleanup task can be performed. * * @return {@link Boolean} */ protected abstract boolean canExecute(); /** - * 全量dump间隔 + * full dump interval. */ static final int DUMP_ALL_INTERVAL_IN_MINUTE = 6 * 60; /** - * 全量dump间隔 + * full dump delay. */ static final int INITIAL_DELAY_IN_MINUTE = 6 * 60; @@ -453,7 +457,7 @@ public void run() { private TaskManager dumpAllTaskMgr; - private static final Logger log = LoggerFactory.getLogger(DumpService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DumpService.class); static final AtomicInteger FINISHED = new AtomicInteger(); @@ -461,11 +465,11 @@ public void run() { int total = 0; - private final static String TRUE_STR = "true"; + private static final String TRUE_STR = "true"; - private final static String BETA_TABLE_NAME = "config_info_beta"; + private static final String BETA_TABLE_NAME = "config_info_beta"; - private final static String TAG_TABLE_NAME = "config_info_tag"; + private static final String TAG_TABLE_NAME = "config_info_tag"; Boolean isQuickStart = false; diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpTask.java deleted file mode 100755 index 85d180e4a93..00000000000 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpTask.java +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Copyright 1999-2018 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.nacos.config.server.service.dump; - -import com.alibaba.nacos.common.utils.MD5Utils; -import com.alibaba.nacos.config.server.constant.Constants; -import com.alibaba.nacos.config.server.manager.AbstractTask; -import com.alibaba.nacos.config.server.manager.TaskProcessor; -import com.alibaba.nacos.config.server.model.ConfigInfo; -import com.alibaba.nacos.config.server.model.ConfigInfo4Beta; -import com.alibaba.nacos.config.server.model.ConfigInfo4Tag; -import com.alibaba.nacos.config.server.model.ConfigInfoBetaWrapper; -import com.alibaba.nacos.config.server.model.ConfigInfoTagWrapper; -import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; -import com.alibaba.nacos.config.server.model.Page; -import com.alibaba.nacos.config.server.model.event.ConfigDumpEvent; -import com.alibaba.nacos.config.server.service.*; -import com.alibaba.nacos.config.server.service.repository.PersistService; -import com.alibaba.nacos.config.server.utils.GroupKey2; -import com.alibaba.nacos.config.server.utils.LogUtil; -import org.apache.commons.lang3.StringUtils; - -import java.sql.Timestamp; -import java.util.List; -import java.util.Objects; - -import static com.alibaba.nacos.config.server.utils.LogUtil.defaultLog; - -/** - * Dump data task - * - * @author Nacos - */ -public class DumpTask extends AbstractTask { - - public DumpTask(String groupKey, long lastModified, String handleIp) { - this.groupKey = groupKey; - this.lastModified = lastModified; - this.handleIp = handleIp; - this.isBeta = false; - this.tag = null; - /** - * retry interval: 1s - */ - setTaskInterval(1000L); - } - - public DumpTask(String groupKey, long lastModified, String handleIp, boolean isBeta) { - this.groupKey = groupKey; - this.lastModified = lastModified; - this.handleIp = handleIp; - this.isBeta = isBeta; - this.tag = null; - /** - * retry interval: 1s - */ - setTaskInterval(1000L); - } - - public DumpTask(String groupKey, String tag, long lastModified, String handleIp, boolean isBeta) { - this.groupKey = groupKey; - this.lastModified = lastModified; - this.handleIp = handleIp; - this.isBeta = isBeta; - this.tag = tag; - /** - * retry interval: 1s - */ - setTaskInterval(1000L); - } - - @Override - public void merge(AbstractTask task) { - } - - final String groupKey; - - final long lastModified; - - final String handleIp; - - final boolean isBeta; - - final String tag; -} - -class DumpAllTask extends AbstractTask { - - @Override - public void merge(AbstractTask task) { - } - - static final String TASK_ID = "dumpAllConfigTask"; -} - -class DumpAllBetaTask extends AbstractTask { - - @Override - public void merge(AbstractTask task) { - } - - static final String TASK_ID = "dumpAllBetaConfigTask"; -} - -class DumpAllTagTask extends AbstractTask { - - @Override - public void merge(AbstractTask task) { - } - - static final String TASK_ID = "dumpAllTagConfigTask"; -} - -class DumpChangeTask extends AbstractTask { - - @Override - public void merge(AbstractTask task) { - } - - static final String TASK_ID = "dumpChangeConfigTask"; -} - -class DumpProcessor implements TaskProcessor { - - DumpProcessor(DumpService dumpService) { - this.dumpService = dumpService; - } - - @Override - public boolean process(String taskType, AbstractTask task) { - final PersistService persistService = dumpService.getPersistService(); - DumpTask dumpTask = (DumpTask) task; - String[] pair = GroupKey2.parseKey(dumpTask.groupKey); - String dataId = pair[0]; - String group = pair[1]; - String tenant = pair[2]; - long lastModified = dumpTask.lastModified; - String handleIp = dumpTask.handleIp; - boolean isBeta = dumpTask.isBeta; - String tag = dumpTask.tag; - - ConfigDumpEvent.ConfigDumpEventBuilder build = ConfigDumpEvent.builder().namespaceId(tenant).dataId(dataId) - .group(group).isBeta(isBeta).tag(tag).lastModifiedTs(lastModified).handleIp(handleIp); - - if (isBeta) { - // beta发布，则dump数据，更新beta缓存 - ConfigInfo4Beta cf = persistService.findConfigInfo4Beta(dataId, group, tenant); - - build.remove(Objects.isNull(cf)); - build.betaIps(Objects.isNull(cf) ? null : cf.getBetaIps()); - build.content(Objects.isNull(cf) ? null : cf.getContent()); - - return DumpConfigHandler.configDump(build.build()); - } else { - if (StringUtils.isBlank(tag)) { - ConfigInfo cf = persistService.findConfigInfo(dataId, group, tenant); - - build.remove(Objects.isNull(cf)); - build.content(Objects.isNull(cf) ? null : cf.getContent()); - build.type(Objects.isNull(cf) ? null : cf.getType()); - - return DumpConfigHandler.configDump(build.build()); - } else { - - ConfigInfo4Tag cf = persistService.findConfigInfo4Tag(dataId, group, tenant, tag); - - build.remove(Objects.isNull(cf)); - build.content(Objects.isNull(cf) ? null : cf.getContent()); - - return DumpConfigHandler.configDump(build.build()); - } - } - } - - final DumpService dumpService; -} - -class DumpAllProcessor implements TaskProcessor { - - DumpAllProcessor(DumpService dumpService) { - this.dumpService = dumpService; - this.persistService = dumpService.getPersistService(); - } - - @Override - public boolean process(String taskType, AbstractTask task) { - long currentMaxId = persistService.findConfigMaxId(); - long lastMaxId = 0; - while (lastMaxId < currentMaxId) { - Page page = persistService.findAllConfigInfoFragment(lastMaxId, PAGE_SIZE); - if (page != null && page.getPageItems() != null && !page.getPageItems().isEmpty()) { - for (ConfigInfoWrapper cf : page.getPageItems()) { - long id = cf.getId(); - lastMaxId = id > lastMaxId ? id : lastMaxId; - if (cf.getDataId().equals(AggrWhitelist.AGGRIDS_METADATA)) { - AggrWhitelist.load(cf.getContent()); - } - - if (cf.getDataId().equals(ClientIpWhiteList.CLIENT_IP_WHITELIST_METADATA)) { - ClientIpWhiteList.load(cf.getContent()); - } - - if (cf.getDataId().equals(SwitchService.SWITCH_META_DATAID)) { - SwitchService.load(cf.getContent()); - } - - boolean result = ConfigCacheService - .dump(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(), cf.getLastModified(), - cf.getType()); - - final String content = cf.getContent(); - final String md5 = MD5Utils.md5Hex(content, Constants.ENCODE); - LogUtil.dumpLog.info("[dump-all-ok] {}, {}, length={}, md5={}", - GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), content.length(), - md5); - } - defaultLog.info("[all-dump] {} / {}", lastMaxId, currentMaxId); - } else { - lastMaxId += PAGE_SIZE; - } - } - return true; - } - - static final int PAGE_SIZE = 1000; - - final DumpService dumpService; - - final PersistService persistService; -} - -class DumpAllBetaProcessor implements TaskProcessor { - - DumpAllBetaProcessor(DumpService dumpService) { - this.dumpService = dumpService; - this.persistService = dumpService.getPersistService(); - } - - @Override - public boolean process(String taskType, AbstractTask task) { - int rowCount = persistService.configInfoBetaCount(); - int pageCount = (int) Math.ceil(rowCount * 1.0 / PAGE_SIZE); - - int actualRowCount = 0; - for (int pageNo = 1; pageNo <= pageCount; pageNo++) { - Page page = persistService.findAllConfigInfoBetaForDumpAll(pageNo, PAGE_SIZE); - if (page != null) { - for (ConfigInfoBetaWrapper cf : page.getPageItems()) { - boolean result = ConfigCacheService - .dumpBeta(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(), - cf.getLastModified(), cf.getBetaIps()); - LogUtil.dumpLog.info("[dump-all-beta-ok] result={}, {}, {}, length={}, md5={}", result, - GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), - cf.getContent().length(), cf.getMd5()); - } - - actualRowCount += page.getPageItems().size(); - defaultLog.info("[all-dump-beta] {} / {}", actualRowCount, rowCount); - } - } - return true; - } - - static final int PAGE_SIZE = 1000; - - final DumpService dumpService; - - final PersistService persistService; -} - -class DumpAllTagProcessor implements TaskProcessor { - - DumpAllTagProcessor(DumpService dumpService) { - this.dumpService = dumpService; - this.persistService = dumpService.getPersistService(); - } - - @Override - public boolean process(String taskType, AbstractTask task) { - int rowCount = persistService.configInfoTagCount(); - int pageCount = (int) Math.ceil(rowCount * 1.0 / PAGE_SIZE); - - int actualRowCount = 0; - for (int pageNo = 1; pageNo <= pageCount; pageNo++) { - Page page = persistService.findAllConfigInfoTagForDumpAll(pageNo, PAGE_SIZE); - if (page != null) { - for (ConfigInfoTagWrapper cf : page.getPageItems()) { - boolean result = ConfigCacheService - .dumpTag(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getTag(), cf.getContent(), - cf.getLastModified()); - LogUtil.dumpLog.info("[dump-all-Tag-ok] result={}, {}, {}, length={}, md5={}", result, - GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), - cf.getContent().length(), cf.getMd5()); - } - - actualRowCount += page.getPageItems().size(); - defaultLog.info("[all-dump-tag] {} / {}", actualRowCount, rowCount); - } - } - return true; - } - - static final int PAGE_SIZE = 1000; - - final DumpService dumpService; - - final PersistService persistService; -} - -class DumpChangeProcessor implements TaskProcessor { - - DumpChangeProcessor(DumpService dumpService, Timestamp startTime, Timestamp endTime) { - this.dumpService = dumpService; - this.persistService = dumpService.getPersistService(); - this.startTime = startTime; - this.endTime = endTime; - } - - @Override - public boolean process(String taskType, AbstractTask task) { - LogUtil.defaultLog.warn("quick start; startTime:{},endTime:{}", startTime, endTime); - LogUtil.defaultLog.warn("updateMd5 start"); - long startUpdateMd5 = System.currentTimeMillis(); - List updateMd5List = persistService.listAllGroupKeyMd5(); - LogUtil.defaultLog.warn("updateMd5 count:{}", updateMd5List.size()); - for (ConfigInfoWrapper config : updateMd5List) { - final String groupKey = GroupKey2.getKey(config.getDataId(), config.getGroup()); - ConfigCacheService.updateMd5(groupKey, config.getMd5(), config.getLastModified()); - } - long endUpdateMd5 = System.currentTimeMillis(); - LogUtil.defaultLog.warn("updateMd5 done,cost:{}", endUpdateMd5 - startUpdateMd5); - - LogUtil.defaultLog.warn("deletedConfig start"); - long startDeletedConfigTime = System.currentTimeMillis(); - List configDeleted = persistService.findDeletedConfig(startTime, endTime); - LogUtil.defaultLog.warn("deletedConfig count:{}", configDeleted.size()); - for (ConfigInfo configInfo : configDeleted) { - if (persistService.findConfigInfo(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant()) - == null) { - ConfigCacheService.remove(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant()); - } - } - long endDeletedConfigTime = System.currentTimeMillis(); - LogUtil.defaultLog.warn("deletedConfig done,cost:{}", endDeletedConfigTime - startDeletedConfigTime); - - LogUtil.defaultLog.warn("changeConfig start"); - long startChangeConfigTime = System.currentTimeMillis(); - List changeConfigs = persistService.findChangeConfig(startTime, endTime); - LogUtil.defaultLog.warn("changeConfig count:{}", changeConfigs.size()); - for (ConfigInfoWrapper cf : changeConfigs) { - boolean result = ConfigCacheService - .dumpChange(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(), cf.getLastModified()); - final String content = cf.getContent(); - final String md5 = MD5Utils.md5Hex(content, Constants.ENCODE); - LogUtil.defaultLog.info("[dump-change-ok] {}, {}, length={}, md5={}", - new Object[] {GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), - content.length(), md5}); - } - ConfigCacheService.reloadConfig(); - long endChangeConfigTime = System.currentTimeMillis(); - LogUtil.defaultLog.warn("changeConfig done,cost:{}", endChangeConfigTime - startChangeConfigTime); - return true; - } - - // ===================== - - final DumpService dumpService; - - final PersistService persistService; - - final Timestamp startTime; - - final Timestamp endTime; -} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/EmbeddedDumpService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/EmbeddedDumpService.java index 50ee3be2de9..f3d645ae12a 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/EmbeddedDumpService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/EmbeddedDumpService.java @@ -28,7 +28,6 @@ import com.alibaba.nacos.consistency.cp.MetadataKey; import com.alibaba.nacos.core.cluster.ServerMemberManager; import com.alibaba.nacos.core.distributed.ProtocolManager; -import com.alibaba.nacos.core.distributed.raft.exception.NoSuchRaftGroupException; import com.alibaba.nacos.core.utils.ApplicationUtils; import com.alibaba.nacos.core.utils.GlobalExecutor; import org.springframework.context.annotation.Conditional; @@ -40,6 +39,8 @@ import java.util.concurrent.atomic.AtomicReference; /** + * Embedded dump service. + * * @author liaochuntao */ @Conditional(ConditionOnEmbeddedStorage.class) @@ -49,18 +50,18 @@ public class EmbeddedDumpService extends DumpService { private final ProtocolManager protocolManager; /** - * If it's just a normal reading failure, it can be resolved by retrying + * If it's just a normal reading failure, it can be resolved by retrying. */ final String[] retryMessages = new String[] {"The conformance protocol is temporarily unavailable for reading"}; /** - * If the read failed due to an internal problem in the Raft state machine, it cannot be remedied by retrying + * If the read failed due to an internal problem in the Raft state machine, it cannot be remedied by retrying. */ final String[] errorMessages = new String[] {"FSMCaller is overload.", "STATE_ERROR"}; /** * Here you inject the dependent objects constructively, ensuring that some of the dependent functionality is - * initialized ahead of time + * initialized ahead of time. * * @param persistService {@link PersistService} * @param memberManager {@link ServerMemberManager} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/ExternalDumpService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/ExternalDumpService.java index c3859a3090f..cf5abf63405 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/ExternalDumpService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/ExternalDumpService.java @@ -25,6 +25,8 @@ import javax.annotation.PostConstruct; /** + * External dump service. + * * @author liaochuntao */ @Conditional(ConditionOnExternalStorage.class) @@ -33,7 +35,7 @@ public class ExternalDumpService extends DumpService { /** * Here you inject the dependent objects constructively, ensuring that some of the dependent functionality is - * initialized ahead of time + * initialized ahead of time. * * @param persistService {@link PersistService} * @param memberManager {@link ServerMemberManager} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllBetaProcessor.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllBetaProcessor.java new file mode 100644 index 00000000000..25231f79915 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllBetaProcessor.java @@ -0,0 +1,75 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.processor; + +import com.alibaba.nacos.config.server.manager.AbstractTask; +import com.alibaba.nacos.config.server.manager.TaskProcessor; +import com.alibaba.nacos.config.server.model.ConfigInfoBetaWrapper; +import com.alibaba.nacos.config.server.model.Page; +import com.alibaba.nacos.config.server.service.ConfigCacheService; +import com.alibaba.nacos.config.server.service.dump.DumpService; +import com.alibaba.nacos.config.server.service.repository.PersistService; +import com.alibaba.nacos.config.server.utils.GroupKey2; +import com.alibaba.nacos.config.server.utils.LogUtil; + +import static com.alibaba.nacos.config.server.utils.LogUtil.defaultLog; + +/** + * Dump all beta processor. + * + * @author Nacos + * @author Wei.Wang + * @date 2020/7/5 12:18 PM + */ +public class DumpAllBetaProcessor implements TaskProcessor { + + public DumpAllBetaProcessor(DumpService dumpService) { + this.dumpService = dumpService; + this.persistService = dumpService.getPersistService(); + } + + @Override + public boolean process(String taskType, AbstractTask task) { + int rowCount = persistService.configInfoBetaCount(); + int pageCount = (int) Math.ceil(rowCount * 1.0 / PAGE_SIZE); + + int actualRowCount = 0; + for (int pageNo = 1; pageNo <= pageCount; pageNo++) { + Page page = persistService.findAllConfigInfoBetaForDumpAll(pageNo, PAGE_SIZE); + if (page != null) { + for (ConfigInfoBetaWrapper cf : page.getPageItems()) { + boolean result = ConfigCacheService + .dumpBeta(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(), + cf.getLastModified(), cf.getBetaIps()); + LogUtil.dumpLog.info("[dump-all-beta-ok] result={}, {}, {}, length={}, md5={}", result, + GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), + cf.getContent().length(), cf.getMd5()); + } + + actualRowCount += page.getPageItems().size(); + defaultLog.info("[all-dump-beta] {} / {}", actualRowCount, rowCount); + } + } + return true; + } + + static final int PAGE_SIZE = 1000; + + final DumpService dumpService; + + final PersistService persistService; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllProcessor.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllProcessor.java new file mode 100644 index 00000000000..24fca0fc42a --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllProcessor.java @@ -0,0 +1,94 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.processor; + +import com.alibaba.nacos.common.utils.MD5Utils; +import com.alibaba.nacos.config.server.constant.Constants; +import com.alibaba.nacos.config.server.manager.AbstractTask; +import com.alibaba.nacos.config.server.manager.TaskProcessor; +import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; +import com.alibaba.nacos.config.server.model.Page; +import com.alibaba.nacos.config.server.service.AggrWhitelist; +import com.alibaba.nacos.config.server.service.ClientIpWhiteList; +import com.alibaba.nacos.config.server.service.ConfigCacheService; +import com.alibaba.nacos.config.server.service.SwitchService; +import com.alibaba.nacos.config.server.service.dump.DumpService; +import com.alibaba.nacos.config.server.service.repository.PersistService; +import com.alibaba.nacos.config.server.utils.GroupKey2; +import com.alibaba.nacos.config.server.utils.LogUtil; + +import static com.alibaba.nacos.config.server.utils.LogUtil.defaultLog; + +/** + * Dump all processor. + * + * @author Nacos + * @date 2020/7/5 12:19 PM + */ +public class DumpAllProcessor implements TaskProcessor { + + public DumpAllProcessor(DumpService dumpService) { + this.dumpService = dumpService; + this.persistService = dumpService.getPersistService(); + } + + @Override + public boolean process(String taskType, AbstractTask task) { + long currentMaxId = persistService.findConfigMaxId(); + long lastMaxId = 0; + while (lastMaxId < currentMaxId) { + Page page = persistService.findAllConfigInfoFragment(lastMaxId, PAGE_SIZE); + if (page != null && page.getPageItems() != null && !page.getPageItems().isEmpty()) { + for (ConfigInfoWrapper cf : page.getPageItems()) { + long id = cf.getId(); + lastMaxId = id > lastMaxId ? id : lastMaxId; + if (cf.getDataId().equals(AggrWhitelist.AGGRIDS_METADATA)) { + AggrWhitelist.load(cf.getContent()); + } + + if (cf.getDataId().equals(ClientIpWhiteList.CLIENT_IP_WHITELIST_METADATA)) { + ClientIpWhiteList.load(cf.getContent()); + } + + if (cf.getDataId().equals(SwitchService.SWITCH_META_DATAID)) { + SwitchService.load(cf.getContent()); + } + + boolean result = ConfigCacheService + .dump(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(), cf.getLastModified(), + cf.getType()); + + final String content = cf.getContent(); + final String md5 = MD5Utils.md5Hex(content, Constants.ENCODE); + LogUtil.dumpLog.info("[dump-all-ok] {}, {}, length={}, md5={}", + GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), content.length(), + md5); + } + defaultLog.info("[all-dump] {} / {}", lastMaxId, currentMaxId); + } else { + lastMaxId += PAGE_SIZE; + } + } + return true; + } + + static final int PAGE_SIZE = 1000; + + final DumpService dumpService; + + final PersistService persistService; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllTagProcessor.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllTagProcessor.java new file mode 100644 index 00000000000..cdd60c3e028 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllTagProcessor.java @@ -0,0 +1,74 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.processor; + +import com.alibaba.nacos.config.server.manager.AbstractTask; +import com.alibaba.nacos.config.server.manager.TaskProcessor; +import com.alibaba.nacos.config.server.model.ConfigInfoTagWrapper; +import com.alibaba.nacos.config.server.model.Page; +import com.alibaba.nacos.config.server.service.ConfigCacheService; +import com.alibaba.nacos.config.server.service.dump.DumpService; +import com.alibaba.nacos.config.server.service.repository.PersistService; +import com.alibaba.nacos.config.server.utils.GroupKey2; +import com.alibaba.nacos.config.server.utils.LogUtil; + +import static com.alibaba.nacos.config.server.utils.LogUtil.defaultLog; + +/** + * Dump all tag processor. + * + * @author Nacos + * @date 2020/7/5 12:18 PM + */ +public class DumpAllTagProcessor implements TaskProcessor { + + public DumpAllTagProcessor(DumpService dumpService) { + this.dumpService = dumpService; + this.persistService = dumpService.getPersistService(); + } + + @Override + public boolean process(String taskType, AbstractTask task) { + int rowCount = persistService.configInfoTagCount(); + int pageCount = (int) Math.ceil(rowCount * 1.0 / PAGE_SIZE); + + int actualRowCount = 0; + for (int pageNo = 1; pageNo <= pageCount; pageNo++) { + Page page = persistService.findAllConfigInfoTagForDumpAll(pageNo, PAGE_SIZE); + if (page != null) { + for (ConfigInfoTagWrapper cf : page.getPageItems()) { + boolean result = ConfigCacheService + .dumpTag(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getTag(), cf.getContent(), + cf.getLastModified()); + LogUtil.dumpLog.info("[dump-all-Tag-ok] result={}, {}, {}, length={}, md5={}", result, + GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), + cf.getContent().length(), cf.getMd5()); + } + + actualRowCount += page.getPageItems().size(); + defaultLog.info("[all-dump-tag] {} / {}", actualRowCount, rowCount); + } + } + return true; + } + + static final int PAGE_SIZE = 1000; + + final DumpService dumpService; + + final PersistService persistService; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpChangeProcessor.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpChangeProcessor.java new file mode 100644 index 00000000000..c7ae6e075e8 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpChangeProcessor.java @@ -0,0 +1,104 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.processor; + +import com.alibaba.nacos.common.utils.MD5Utils; +import com.alibaba.nacos.config.server.constant.Constants; +import com.alibaba.nacos.config.server.manager.AbstractTask; +import com.alibaba.nacos.config.server.manager.TaskProcessor; +import com.alibaba.nacos.config.server.model.ConfigInfo; +import com.alibaba.nacos.config.server.model.ConfigInfoWrapper; +import com.alibaba.nacos.config.server.service.ConfigCacheService; +import com.alibaba.nacos.config.server.service.dump.DumpService; +import com.alibaba.nacos.config.server.service.repository.PersistService; +import com.alibaba.nacos.config.server.utils.GroupKey2; +import com.alibaba.nacos.config.server.utils.LogUtil; + +import java.sql.Timestamp; +import java.util.List; + +/** + * Dump change processor. + * + * @author Nacos + * @date 2020/7/5 12:19 PM + */ +public class DumpChangeProcessor implements TaskProcessor { + + public DumpChangeProcessor(DumpService dumpService, Timestamp startTime, Timestamp endTime) { + this.dumpService = dumpService; + this.persistService = dumpService.getPersistService(); + this.startTime = startTime; + this.endTime = endTime; + } + + @Override + public boolean process(String taskType, AbstractTask task) { + LogUtil.defaultLog.warn("quick start; startTime:{},endTime:{}", startTime, endTime); + LogUtil.defaultLog.warn("updateMd5 start"); + long startUpdateMd5 = System.currentTimeMillis(); + List updateMd5List = persistService.listAllGroupKeyMd5(); + LogUtil.defaultLog.warn("updateMd5 count:{}", updateMd5List.size()); + for (ConfigInfoWrapper config : updateMd5List) { + final String groupKey = GroupKey2.getKey(config.getDataId(), config.getGroup()); + ConfigCacheService.updateMd5(groupKey, config.getMd5(), config.getLastModified()); + } + long endUpdateMd5 = System.currentTimeMillis(); + LogUtil.defaultLog.warn("updateMd5 done,cost:{}", endUpdateMd5 - startUpdateMd5); + + LogUtil.defaultLog.warn("deletedConfig start"); + long startDeletedConfigTime = System.currentTimeMillis(); + List configDeleted = persistService.findDeletedConfig(startTime, endTime); + LogUtil.defaultLog.warn("deletedConfig count:{}", configDeleted.size()); + for (ConfigInfo configInfo : configDeleted) { + if (persistService.findConfigInfo(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant()) + == null) { + ConfigCacheService.remove(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant()); + } + } + long endDeletedConfigTime = System.currentTimeMillis(); + LogUtil.defaultLog.warn("deletedConfig done,cost:{}", endDeletedConfigTime - startDeletedConfigTime); + + LogUtil.defaultLog.warn("changeConfig start"); + final long startChangeConfigTime = System.currentTimeMillis(); + List changeConfigs = persistService.findChangeConfig(startTime, endTime); + LogUtil.defaultLog.warn("changeConfig count:{}", changeConfigs.size()); + for (ConfigInfoWrapper cf : changeConfigs) { + boolean result = ConfigCacheService + .dumpChange(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(), cf.getLastModified()); + final String content = cf.getContent(); + final String md5 = MD5Utils.md5Hex(content, Constants.ENCODE); + LogUtil.defaultLog.info("[dump-change-ok] {}, {}, length={}, md5={}", + new Object[] {GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), + content.length(), md5}); + } + ConfigCacheService.reloadConfig(); + long endChangeConfigTime = System.currentTimeMillis(); + LogUtil.defaultLog.warn("changeConfig done,cost:{}", endChangeConfigTime - startChangeConfigTime); + return true; + } + + // ===================== + + final DumpService dumpService; + + final PersistService persistService; + + final Timestamp startTime; + + final Timestamp endTime; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpProcessor.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpProcessor.java new file mode 100644 index 00000000000..963ef860005 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpProcessor.java @@ -0,0 +1,93 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.processor; + +import com.alibaba.nacos.config.server.manager.AbstractTask; +import com.alibaba.nacos.config.server.manager.TaskProcessor; +import com.alibaba.nacos.config.server.model.ConfigInfo; +import com.alibaba.nacos.config.server.model.ConfigInfo4Beta; +import com.alibaba.nacos.config.server.model.ConfigInfo4Tag; +import com.alibaba.nacos.config.server.model.event.ConfigDumpEvent; +import com.alibaba.nacos.config.server.service.dump.DumpConfigHandler; +import com.alibaba.nacos.config.server.service.dump.DumpService; +import com.alibaba.nacos.config.server.service.dump.task.DumpTask; +import com.alibaba.nacos.config.server.service.repository.PersistService; +import com.alibaba.nacos.config.server.utils.GroupKey2; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; + +/** + * dump processor. + * + * @author Nacos + * @date 2020/7/5 12:19 PM + */ +public class DumpProcessor implements TaskProcessor { + + public DumpProcessor(DumpService dumpService) { + this.dumpService = dumpService; + } + + @Override + public boolean process(String taskType, AbstractTask task) { + final PersistService persistService = dumpService.getPersistService(); + DumpTask dumpTask = (DumpTask) task; + String[] pair = GroupKey2.parseKey(dumpTask.getGroupKey()); + String dataId = pair[0]; + String group = pair[1]; + String tenant = pair[2]; + long lastModified = dumpTask.getLastModified(); + String handleIp = dumpTask.getHandleIp(); + boolean isBeta = dumpTask.isBeta(); + String tag = dumpTask.getTag(); + + ConfigDumpEvent.ConfigDumpEventBuilder build = ConfigDumpEvent.builder().namespaceId(tenant).dataId(dataId) + .group(group).isBeta(isBeta).tag(tag).lastModifiedTs(lastModified).handleIp(handleIp); + + if (isBeta) { + // beta发布，则dump数据，更新beta缓存 + ConfigInfo4Beta cf = persistService.findConfigInfo4Beta(dataId, group, tenant); + + build.remove(Objects.isNull(cf)); + build.betaIps(Objects.isNull(cf) ? null : cf.getBetaIps()); + build.content(Objects.isNull(cf) ? null : cf.getContent()); + + return DumpConfigHandler.configDump(build.build()); + } else { + if (StringUtils.isBlank(tag)) { + ConfigInfo cf = persistService.findConfigInfo(dataId, group, tenant); + + build.remove(Objects.isNull(cf)); + build.content(Objects.isNull(cf) ? null : cf.getContent()); + build.type(Objects.isNull(cf) ? null : cf.getType()); + + return DumpConfigHandler.configDump(build.build()); + } else { + + ConfigInfo4Tag cf = persistService.findConfigInfo4Tag(dataId, group, tenant, tag); + + build.remove(Objects.isNull(cf)); + build.content(Objects.isNull(cf) ? null : cf.getContent()); + + return DumpConfigHandler.configDump(build.build()); + } + } + } + + final DumpService dumpService; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllBetaTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllBetaTask.java new file mode 100644 index 00000000000..21d3e99b60e --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllBetaTask.java @@ -0,0 +1,34 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.task; + +import com.alibaba.nacos.config.server.manager.AbstractTask; + +/** + * Dump all beta task. + * + * @author Nacos + * @date 2020/7/5 12:19 PM + */ +public class DumpAllBetaTask extends AbstractTask { + + @Override + public void merge(AbstractTask task) { + } + + public static final String TASK_ID = "dumpAllBetaConfigTask"; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTagTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTagTask.java new file mode 100644 index 00000000000..ec1121751a7 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTagTask.java @@ -0,0 +1,34 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.task; + +import com.alibaba.nacos.config.server.manager.AbstractTask; + +/** + * Dump all tag task. + * + * @author Nacos + * @date 2020/7/5 12:19 PM + */ +public class DumpAllTagTask extends AbstractTask { + + @Override + public void merge(AbstractTask task) { + } + + public static final String TASK_ID = "dumpAllTagConfigTask"; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTask.java new file mode 100644 index 00000000000..ee6e6e8fb7d --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTask.java @@ -0,0 +1,34 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.task; + +import com.alibaba.nacos.config.server.manager.AbstractTask; + +/** + * Dump all task. + * + * @author Nacos + * @date 2020/7/5 12:17 PM + */ +public class DumpAllTask extends AbstractTask { + + @Override + public void merge(AbstractTask task) { + } + + public static final String TASK_ID = "dumpAllConfigTask"; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpChangeTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpChangeTask.java new file mode 100644 index 00000000000..948dffc31af --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpChangeTask.java @@ -0,0 +1,34 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.task; + +import com.alibaba.nacos.config.server.manager.AbstractTask; + +/** + * Dump change task. + * + * @author Nacos + * @date 2020/7/5 12:19 PM + */ +public class DumpChangeTask extends AbstractTask { + + @Override + public void merge(AbstractTask task) { + } + + public static final String TASK_ID = "dumpChangeConfigTask"; +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpTask.java new file mode 100755 index 00000000000..ce2aa240ba7 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpTask.java @@ -0,0 +1,94 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.service.dump.task; + +import com.alibaba.nacos.config.server.manager.AbstractTask; + +/** + * Dump data task. + * + * @author Nacos + */ +public class DumpTask extends AbstractTask { + + public DumpTask(String groupKey, long lastModified, String handleIp) { + this.groupKey = groupKey; + this.lastModified = lastModified; + this.handleIp = handleIp; + this.isBeta = false; + this.tag = null; + // retry interval: 1s + setTaskInterval(1000L); + } + + public DumpTask(String groupKey, long lastModified, String handleIp, boolean isBeta) { + this.groupKey = groupKey; + this.lastModified = lastModified; + this.handleIp = handleIp; + this.isBeta = isBeta; + this.tag = null; + // retry interval: 1s + setTaskInterval(1000L); + } + + public DumpTask(String groupKey, String tag, long lastModified, String handleIp, boolean isBeta) { + this.groupKey = groupKey; + this.lastModified = lastModified; + this.handleIp = handleIp; + this.isBeta = isBeta; + this.tag = tag; + // retry interval: 1s + setTaskInterval(1000L); + } + + @Override + public void merge(AbstractTask task) { + } + + final String groupKey; + + final long lastModified; + + final String handleIp; + + final boolean isBeta; + + final String tag; + + + public String getGroupKey() { + return groupKey; + } + + public long getLastModified() { + return lastModified; + } + + public String getHandleIp() { + return handleIp; + } + + public boolean isBeta() { + return isBeta; + } + + public String getTag() { + return tag; + } + +} + diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDataTask.java b/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDataTask.java index dfa5f35ed11..9242822f1f7 100755 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDataTask.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDataTask.java @@ -19,7 +19,7 @@ import com.alibaba.nacos.config.server.manager.AbstractTask; /** - * 表示对数据进行聚合的任务。 + * Represents the task of aggregating data. * * @author jiuRen */ @@ -36,7 +36,7 @@ class MergeDataTask extends AbstractTask { this.tag = tag; this.clientIp = clientIp; - // 聚合延迟 + // aggregation delay setTaskInterval(DELAY); setLastProcessTime(System.currentTimeMillis()); } diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDatumService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDatumService.java index c72bfdf4071..61daddd3e6a 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDatumService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeDatumService.java @@ -26,9 +26,7 @@ import com.alibaba.nacos.config.server.utils.ContentUtils; import com.alibaba.nacos.config.server.utils.PropertyUtil; import com.alibaba.nacos.config.server.utils.TimeUtils; -import com.alibaba.nacos.consistency.cp.CPProtocol; import com.alibaba.nacos.core.distributed.ProtocolManager; -import com.alibaba.nacos.core.distributed.raft.exception.NoSuchRaftGroupException; import com.alibaba.nacos.core.utils.ApplicationUtils; import com.alibaba.nacos.core.utils.InetUtils; import org.slf4j.Logger; @@ -42,9 +40,9 @@ import java.util.concurrent.atomic.AtomicInteger; /** - * 数据聚合服务。 - *

- * 启动时做全量聚合 + 修改数据触发的单条聚合 + * Data aggregation service. + * + *

Full aggregation at startup and single aggregation triggered by data modification. * * @author jiuRen */ @@ -79,7 +77,7 @@ static List> splitList(List list, int } /** - * 数据变更后调用，添加聚合任务 + * Called after data changes to add aggregation tasks. */ public void addMergeTask(String dataId, String groupId, String tenant, String tag, String clientIp) { if (!canExecute()) { @@ -90,7 +88,7 @@ public void addMergeTask(String dataId, String groupId, String tenant, String ta } /** - * 数据变更后调用，添加聚合任务 + * Called after data changes to add aggregation tasks. */ public void addMergeTask(String dataId, String groupId, String tenant, String clientIp) { if (!canExecute()) { @@ -100,6 +98,9 @@ public void addMergeTask(String dataId, String groupId, String tenant, String cl mergeTasks.addTask(task.getId(), task); } + /** + * Merge all. + */ public void mergeAll() { if (!canExecute()) { return; @@ -146,42 +147,40 @@ public void run() { .findConfigInfoAggrByPage(dataId, group, tenant, pageNo, PAGE_SIZE); if (page != null) { datumList.addAll(page.getPageItems()); - log.info("[merge-query] {}, {}, size/total={}/{}", dataId, group, datumList.size(), + LOGGER.info("[merge-query] {}, {}, size/total={}/{}", dataId, group, datumList.size(), rowCount); } } final Timestamp time = TimeUtils.getCurrentTime(); - // 聚合 + if (datumList.size() > 0) { + // merge ConfigInfo cf = MergeTaskProcessor.merge(dataId, group, tenant, datumList); persistService.insertOrUpdate(null, null, cf, time, null, false); - log.info("[merge-ok] {}, {}, size={}, length={}, md5={}, content={}", dataId, group, + LOGGER.info("[merge-ok] {}, {}, size={}, length={}, md5={}, content={}", dataId, group, datumList.size(), cf.getContent().length(), cf.getMd5(), ContentUtils.truncateContent(cf.getContent())); - } - // 删除 - else { + } else { + // remove persistService.removeConfigInfo(dataId, group, tenant, InetUtils.getSelfIp(), null); - log.warn("[merge-delete] delete config info because no datum. dataId=" + dataId + ", groupId=" + LOGGER.warn("[merge-delete] delete config info because no datum. dataId=" + dataId + ", groupId=" + group); } } catch (Exception e) { - log.info("[merge-error] " + dataId + ", " + group + ", " + e.toString(), e); + LOGGER.info("[merge-error] " + dataId + ", " + group + ", " + e.toString(), e); } FINISHED.incrementAndGet(); if (FINISHED.get() % 100 == 0) { - log.info("[all-merge-dump] {} / {}", FINISHED.get(), total); + LOGGER.info("[all-merge-dump] {} / {}", FINISHED.get(), total); } } - log.info("[all-merge-dump] {} / {}", FINISHED.get(), total); + LOGGER.info("[all-merge-dump] {} / {}", FINISHED.get(), total); } } - // ===================== - - private static final Logger log = LoggerFactory.getLogger(MergeDatumService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MergeDatumService.class); final TaskManager mergeTasks; diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeTaskProcessor.java b/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeTaskProcessor.java index 797c1c7540f..1e9f4fe6d01 100755 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeTaskProcessor.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/merge/MergeTaskProcessor.java @@ -38,13 +38,13 @@ import java.util.List; /** - * Merge task processor + * Merge task processor. * * @author Nacos */ public class MergeTaskProcessor implements TaskProcessor { - final int PAGE_SIZE = 10000; + private static final int PAGE_SIZE = 10000; MergeTaskProcessor(PersistService persistService, MergeDatumService mergeService) { this.persistService = persistService; @@ -68,33 +68,32 @@ public boolean process(String taskType, AbstractTask task) { .findConfigInfoAggrByPage(dataId, group, tenant, pageNo, PAGE_SIZE); if (page != null) { datumList.addAll(page.getPageItems()); - log.info("[merge-query] {}, {}, size/total={}/{}", dataId, group, datumList.size(), rowCount); + LOGGER.info("[merge-query] {}, {}, size/total={}/{}", dataId, group, datumList.size(), rowCount); } } final Timestamp time = TimeUtils.getCurrentTime(); - // 聚合 if (datumList.size() > 0) { + // merge ConfigInfo cf = merge(dataId, group, tenant, datumList); persistService.insertOrUpdate(null, null, cf, time, null); - log.info("[merge-ok] {}, {}, size={}, length={}, md5={}, content={}", dataId, group, datumList.size(), + LOGGER.info("[merge-ok] {}, {}, size={}, length={}, md5={}, content={}", dataId, group, datumList.size(), cf.getContent().length(), cf.getMd5(), ContentUtils.truncateContent(cf.getContent())); ConfigTraceService .logPersistenceEvent(dataId, group, tenant, null, time.getTime(), InetUtils.getSelfIp(), ConfigTraceService.PERSISTENCE_EVENT_MERGE, cf.getContent()); - } - // 删除 - else { + } else { + // remove if (StringUtils.isBlank(tag)) { persistService.removeConfigInfo(dataId, group, tenant, clientIp, null); } else { persistService.removeConfigInfoTag(dataId, group, tenant, tag, clientIp, null); } - log.warn("[merge-delete] delete config info because no datum. dataId=" + dataId + ", groupId=" + group); + LOGGER.warn("[merge-delete] delete config info because no datum. dataId=" + dataId + ", groupId=" + group); ConfigTraceService .logPersistenceEvent(dataId, group, tenant, null, time.getTime(), InetUtils.getSelfIp(), @@ -105,12 +104,21 @@ public boolean process(String taskType, AbstractTask task) { } catch (Exception e) { mergeService.addMergeTask(dataId, group, tenant, mergeTask.getClientIp()); - log.info("[merge-error] " + dataId + ", " + group + ", " + e.toString(), e); + LOGGER.info("[merge-error] " + dataId + ", " + group + ", " + e.toString(), e); } return true; } + /** + * merge datumList {@link ConfigInfoAggr}. + * + * @param dataId data id + * @param group group + * @param tenant tenant + * @param datumList datumList + * @return {@link ConfigInfo} + */ public static ConfigInfo merge(String dataId, String group, String tenant, List datumList) { StringBuilder sb = new StringBuilder(); String appName = null; @@ -125,9 +133,7 @@ public static ConfigInfo merge(String dataId, String group, String tenant, List< return new ConfigInfo(dataId, group, tenant, appName, content); } - // ===================== - - private static final Logger log = LoggerFactory.getLogger(MergeTaskProcessor.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MergeTaskProcessor.class); private PersistService persistService;