From 73fba08206ad4d60e5632314a26104cdd6414306 Mon Sep 17 00:00:00 2001 From: masaimu Date: Tue, 9 Jul 2024 18:22:53 +0800 Subject: [PATCH] fix: agentId stat (#874) --- .../common/service/AlertRuleService.java | 10 +++- .../server/common/service/SuperCache.java | 5 ++ .../common/service/SuperCacheService.java | 41 +++++++++++++++ .../service/impl/AlertRuleServiceImpl.java | 43 ++++++++++++---- .../AlarmRuleLevelAuthorizationChecker.java | 24 +++++---- .../registry/core/agent/AgentService.java | 51 ++++++++++--------- .../grpc/RegistryServiceForAgentImpl.java | 35 ++++++------- 7 files changed, 146 insertions(+), 63 deletions(-) diff --git a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/AlertRuleService.java b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/AlertRuleService.java index c9694bd52..3f33e14ea 100644 --- a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/AlertRuleService.java +++ b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/AlertRuleService.java @@ -4,10 +4,10 @@ package io.holoinsight.server.common.service; import com.baomidou.mybatisplus.extension.service.IService; -import io.holoinsight.server.common.dao.entity.AlarmRule; -import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO; import io.holoinsight.server.common.MonitorPageRequest; import io.holoinsight.server.common.MonitorPageResult; +import io.holoinsight.server.common.dao.entity.AlarmRule; +import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO; import java.util.List; @@ -23,6 +23,12 @@ public interface AlertRuleService extends IService { Boolean deleteById(Long id); + Long save(AlarmRuleDTO alarmRuleDTO, boolean busPost); + + Boolean updateById(AlarmRuleDTO alarmRuleDTO, boolean busPost); + + Boolean deleteById(Long id, boolean busPost); + AlarmRuleDTO queryById(Long id, String tenant, String workspace); List queryBySourceType(String sourceType, String tenant, String workspace); diff --git a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java index f517db34a..6f6f04d8d 100644 --- a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java +++ b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; /** * @@ -25,6 +26,10 @@ public class SuperCache { public List resourceKeys; public List freePrefixes; + public Set metricTypes; + + public Set integrationProducts; + public String getStringValue(String type, String k) { Map kMap = this.metaDataDictValueMap.get(type); diff --git a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java index 0d44dbe2a..5055650be 100644 --- a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java +++ b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java @@ -3,12 +3,22 @@ */ package io.holoinsight.server.common.service; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import io.holoinsight.server.common.config.ProdLog; import io.holoinsight.server.common.config.ScheduleLoadTask; +import io.holoinsight.server.common.dao.entity.IntegrationProduct; +import io.holoinsight.server.common.dao.entity.MetricInfo; +import io.holoinsight.server.common.dao.mapper.IntegrationProductMapper; +import io.holoinsight.server.common.dao.mapper.MetricInfoMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; +import javax.annotation.Resource; import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * @@ -24,6 +34,10 @@ public class SuperCacheService extends ScheduleLoadTask { @Autowired private MetricInfoService metricInfoService; + @Resource + private MetricInfoMapper metricInfoMapper; + @Resource + private IntegrationProductMapper integrationProductMapper; public SuperCache getSc() { return sc; @@ -42,10 +56,37 @@ public void load() throws Exception { sc.getListValue("global_config", "resource_keys", Collections.singletonList("tenant")); sc.freePrefixes = sc.getListValue("global_config", "free_metric_prefix", Collections.emptyList()); + sc.metricTypes = queryMetricTypes(); + sc.integrationProducts = queryIntegrationProducts(); this.sc = sc; ProdLog.info("[SuperCache] load end"); } + private Set queryIntegrationProducts() { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.select("DISTINCT name"); + List integrationProducts = + this.integrationProductMapper.selectList(queryWrapper); + if (CollectionUtils.isEmpty(integrationProducts)) { + return Collections.emptySet(); + } + return integrationProducts.stream() // + .map(IntegrationProduct::getName) // + .collect(Collectors.toSet()); + } + + private Set queryMetricTypes() { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.select("DISTINCT metric_type"); + List metricInfoList = this.metricInfoMapper.selectList(queryWrapper); + if (CollectionUtils.isEmpty(metricInfoList)) { + return Collections.emptySet(); + } + return metricInfoList.stream() // + .map(MetricInfo::getMetricType) // + .collect(Collectors.toSet()); + } + @Override public int periodInSeconds() { return 60; diff --git a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/impl/AlertRuleServiceImpl.java b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/impl/AlertRuleServiceImpl.java index fd0a290ff..d1854a191 100644 --- a/server/common/common-service/src/main/java/io/holoinsight/server/common/service/impl/AlertRuleServiceImpl.java +++ b/server/common/common-service/src/main/java/io/holoinsight/server/common/service/impl/AlertRuleServiceImpl.java @@ -9,21 +9,21 @@ import io.holoinsight.server.common.EventBusHolder; import io.holoinsight.server.common.J; import io.holoinsight.server.common.MD5Hash; +import io.holoinsight.server.common.MonitorPageRequest; +import io.holoinsight.server.common.MonitorPageResult; import io.holoinsight.server.common.dao.converter.AlarmRuleConverter; +import io.holoinsight.server.common.dao.entity.AlarmBlock; +import io.holoinsight.server.common.dao.entity.AlarmRule; +import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO; +import io.holoinsight.server.common.dao.entity.dto.AlertRuleExtra; import io.holoinsight.server.common.dao.entity.dto.alarm.AlarmRuleConf; import io.holoinsight.server.common.dao.entity.dto.alarm.trigger.DataSource; import io.holoinsight.server.common.dao.entity.dto.alarm.trigger.Filter; import io.holoinsight.server.common.dao.entity.dto.alarm.trigger.Trigger; import io.holoinsight.server.common.dao.mapper.AlarmRuleMapper; -import io.holoinsight.server.common.dao.entity.AlarmBlock; -import io.holoinsight.server.common.dao.entity.AlarmRule; -import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO; -import io.holoinsight.server.common.dao.entity.dto.AlertRuleExtra; import io.holoinsight.server.common.service.AlertBlockService; import io.holoinsight.server.common.service.AlertRuleService; import io.holoinsight.server.common.service.RequestContextAdapter; -import io.holoinsight.server.common.MonitorPageRequest; -import io.holoinsight.server.common.MonitorPageResult; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; @@ -57,29 +57,50 @@ public class AlertRuleServiceImpl extends ServiceImpl(Arrays.asList("default", "gradual", "fixed")); private static final Set aggregators = new HashSet<>(Arrays.asList("sum", "avg", "min", "max", "count", "none", "SUM", "AVG", "MIN", "MAX", "COUNT", "NONE")); - private static final Set metricTypes = - new HashSet<>(Arrays.asList("app", "cache", "log", "oss", "trace", "system", "metric", - "service", "function", "pg", "mongodb", "db", "miniProgram", "mysql")); - private static final Set products = new HashSet<>( - Arrays.asList("JVM", "Function", "OceanBase", "Tbase", "PortCheck", "System", "MiniProgram", - "Spanner", "IoT", "APM", "Mysql", "SLB", "SOFAMQX", "Postgres", "Gateway")); + private static final Set defaultMetricTypes = + new HashSet<>(Arrays.asList("message", "loadbalancing")); + // private static final Set products = new HashSet<>( + // Arrays.asList("JVM", "Function", "OceanBase", "Tbase", "PortCheck", "System", "MiniProgram", + // "Spanner", "IoT", "APM", "Mysql", "SLB", "SOFAMQX", "Postgres", "Gateway")); @Override public LevelAuthorizationCheckResult check(LevelAuthorizationMetaData levelAuthMetaData, @@ -423,12 +423,16 @@ private LevelAuthorizationCheckResult checkRule(Map ruleMap, Str private LevelAuthorizationCheckResult checkDatasources(List datasources, String tenant, String workspace) { + Set metricTypes = this.superCacheService.getSc().metricTypes; + Set products = this.superCacheService.getSc().integrationProducts; for (DataSource dataSource : datasources) { if (StringUtils.isNotEmpty(dataSource.getMetricType()) - && !metricTypes.contains(dataSource.getMetricType())) { + && !CollectionUtils.isEmpty(metricTypes) + && !metricTypes.contains(dataSource.getMetricType()) + && !defaultMetricTypes.contains(dataSource.getMetricType())) { return failCheckResult("invalid metric type %s", dataSource.getMetricType()); } - if (StringUtils.isNotEmpty(dataSource.getProduct()) + if (StringUtils.isNotEmpty(dataSource.getProduct()) && !CollectionUtils.isEmpty(products) && !products.contains(dataSource.getProduct())) { return failCheckResult("invalid product %s", dataSource.getProduct()); } diff --git a/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/agent/AgentService.java b/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/agent/AgentService.java index 85cc346da..3fd92c98b 100644 --- a/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/agent/AgentService.java +++ b/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/agent/AgentService.java @@ -3,26 +3,6 @@ */ package io.holoinsight.server.registry.core.agent; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - import com.google.common.collect.Maps; import com.xzchaoo.commons.basic.Ack; import com.xzchaoo.commons.basic.Acks; @@ -31,22 +11,40 @@ import com.xzchaoo.commons.batchprocessor.Flusher; import com.xzchaoo.commons.stat.StatAccumulator; import com.xzchaoo.commons.stat.StringsKey; - import io.holoinsight.server.common.JsonUtils; +import io.holoinsight.server.common.MetricsUtils; import io.holoinsight.server.common.NetUtils; import io.holoinsight.server.common.auth.AuthInfo; import io.holoinsight.server.common.dao.entity.GaeaAgentDO; import io.holoinsight.server.common.dao.entity.GaeaAgentDOExample; import io.holoinsight.server.common.dao.mapper.GaeaAgentDOMapper; +import io.holoinsight.server.common.event.EventBusHolder; import io.holoinsight.server.common.threadpool.CommonThreadPools; import io.holoinsight.server.meta.common.model.QueryExample; import io.holoinsight.server.meta.facade.model.MetaType; import io.holoinsight.server.meta.facade.service.AgentHeartBeatService; import io.holoinsight.server.meta.facade.service.DataClientService; -import io.holoinsight.server.common.event.EventBusHolder; -import io.holoinsight.server.common.MetricsUtils; import io.holoinsight.server.registry.grpc.agent.AgentK8sInfo; import io.holoinsight.server.registry.grpc.agent.RegisterAgentRequest; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; /** *

@@ -60,6 +58,8 @@ public class AgentService { private static final Logger LOGGER = LoggerFactory.getLogger("AGENT"); private static final StatAccumulator AGENT_HEARTBEAT_STAT = MetricsUtils.SM.create("agent.heartbeat"); + private static final StatAccumulator AGENTID_HEARTBEAT_STAT = + MetricsUtils.SM.create("agentId.heartbeat"); @Autowired private GaeaAgentDOMapper mapper; @@ -107,6 +107,11 @@ private void flushHeartbeats(List agentIds) { long end = System.currentTimeMillis(); AGENT_HEARTBEAT_STAT.add(StringsKey.EMPTY, new long[] {1, count, end - begin}); + if (CollectionUtils.isNotEmpty(agentIds)) { + for (String agentId : agentIds) { + AGENTID_HEARTBEAT_STAT.add(StringsKey.of(agentId), new long[] {1}); + } + } } @PreDestroy diff --git a/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/grpc/RegistryServiceForAgentImpl.java b/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/grpc/RegistryServiceForAgentImpl.java index 87aaf7583..a8b818d9f 100644 --- a/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/grpc/RegistryServiceForAgentImpl.java +++ b/server/registry/registry-core/src/main/java/io/holoinsight/server/registry/core/grpc/RegistryServiceForAgentImpl.java @@ -3,22 +3,6 @@ */ package io.holoinsight.server.registry.core.grpc; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.google.protobuf.ByteString; @@ -27,7 +11,6 @@ import com.xzchaoo.commons.stat.StatAccumulator; import com.xzchaoo.commons.stat.Stats; import com.xzchaoo.commons.stat.StringsKey; - import io.grpc.Status; import io.grpc.StatusException; import io.grpc.StatusRuntimeException; @@ -74,8 +57,23 @@ import io.holoinsight.server.registry.grpc.agent.ReportEventRequest; import io.holoinsight.server.registry.grpc.agent.SendAgentHeartbeatRequest; import io.holoinsight.server.registry.grpc.agent.SendAgentHeartbeatResponse; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + /** *

* created at 2022/3/1 @@ -91,6 +89,8 @@ public class RegistryServiceForAgentImpl MetricsUtils.SM1S.create("grpc.agent.heartbeat"); private static final StatAccumulator AGENT_PULLCONFIG_STAT = MetricsUtils.SM.create("agent.pullconfig.stat"); + private static final StatAccumulator AGENTID_PULLCONFIG_STAT = + MetricsUtils.SM.create("agentId.pullconfig.stat"); @Autowired private ServerStreamManager serverStreamManager; @Autowired @@ -432,6 +432,7 @@ private GetCollectTasksResponse getCollectTasks0(GetCollectTasksRequest request) LOGGER.info("agent={} keys={} missDim={} tasks={}", request.getAgentId(), keys.size(), missDim, count); AGENT_PULLCONFIG_STAT.add(StringsKey.of(agent.getTenant()), new long[] {1, count}); + AGENTID_PULLCONFIG_STAT.add(StringsKey.of(agent.getId()), new long[] {1, count}); return respB.build(); }