Skip to content

Commit

Permalink
fix: agentId stat (#874)
Browse files Browse the repository at this point in the history
  • Loading branch information
@masaimu
masaimu authored Jul 9, 2024
1 parent f03b78f commit 73fba08
Show file tree
Hide file tree
Showing 7 changed files with 146 additions and 63 deletions.
10 changes: 8 additions & 2 deletions ...n/common-service/src/main/java/io/holoinsight/server/common/service/AlertRuleService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
package io.holoinsight.server.common.service;

import com.baomidou.mybatisplus.extension.service.IService;
import io.holoinsight.server.common.dao.entity.AlarmRule;
import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO;
import io.holoinsight.server.common.MonitorPageRequest;
import io.holoinsight.server.common.MonitorPageResult;
import io.holoinsight.server.common.dao.entity.AlarmRule;
import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO;

import java.util.List;

Expand All @@ -23,6 +23,12 @@ public interface AlertRuleService extends IService<AlarmRule> {

Boolean deleteById(Long id);

Long save(AlarmRuleDTO alarmRuleDTO, boolean busPost);

Boolean updateById(AlarmRuleDTO alarmRuleDTO, boolean busPost);

Boolean deleteById(Long id, boolean busPost);

AlarmRuleDTO queryById(Long id, String tenant, String workspace);

List<AlarmRuleDTO> queryBySourceType(String sourceType, String tenant, String workspace);
Expand Down
5 changes: 5 additions & 0 deletions .../common/common-service/src/main/java/io/holoinsight/server/common/service/SuperCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import java.util.List;
import java.util.Map;
import java.util.Set;

/**
*
Expand All @@ -25,6 +26,10 @@ public class SuperCache {
public List<String> resourceKeys;
public List<String> freePrefixes;

public Set<String> metricTypes;

public Set<String> integrationProducts;

public String getStringValue(String type, String k) {

Map<String, MetaDataDictValue> kMap = this.metaDataDictValueMap.get(type);
Expand Down
41 changes: 41 additions & 0 deletions .../common-service/src/main/java/io/holoinsight/server/common/service/SuperCacheService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,22 @@
*/
package io.holoinsight.server.common.service;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import io.holoinsight.server.common.config.ProdLog;
import io.holoinsight.server.common.config.ScheduleLoadTask;
import io.holoinsight.server.common.dao.entity.IntegrationProduct;
import io.holoinsight.server.common.dao.entity.MetricInfo;
import io.holoinsight.server.common.dao.mapper.IntegrationProductMapper;
import io.holoinsight.server.common.dao.mapper.MetricInfoMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.CollectionUtils;

import javax.annotation.Resource;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
*
Expand All @@ -24,6 +34,10 @@ public class SuperCacheService extends ScheduleLoadTask {

@Autowired
private MetricInfoService metricInfoService;
@Resource
private MetricInfoMapper metricInfoMapper;
@Resource
private IntegrationProductMapper integrationProductMapper;

public SuperCache getSc() {
return sc;
Expand All @@ -42,10 +56,37 @@ public void load() throws Exception {
sc.getListValue("global_config", "resource_keys", Collections.singletonList("tenant"));
sc.freePrefixes =
sc.getListValue("global_config", "free_metric_prefix", Collections.emptyList());
sc.metricTypes = queryMetricTypes();
sc.integrationProducts = queryIntegrationProducts();
this.sc = sc;
ProdLog.info("[SuperCache] load end");
}

private Set<String> queryIntegrationProducts() {
QueryWrapper<IntegrationProduct> queryWrapper = new QueryWrapper<>();
queryWrapper.select("DISTINCT name");
List<IntegrationProduct> integrationProducts =
this.integrationProductMapper.selectList(queryWrapper);
if (CollectionUtils.isEmpty(integrationProducts)) {
return Collections.emptySet();
}
return integrationProducts.stream() //
.map(IntegrationProduct::getName) //
.collect(Collectors.toSet());
}

private Set<String> queryMetricTypes() {
QueryWrapper<MetricInfo> queryWrapper = new QueryWrapper<>();
queryWrapper.select("DISTINCT metric_type");
List<MetricInfo> metricInfoList = this.metricInfoMapper.selectList(queryWrapper);
if (CollectionUtils.isEmpty(metricInfoList)) {
return Collections.emptySet();
}
return metricInfoList.stream() //
.map(MetricInfo::getMetricType) //
.collect(Collectors.toSet());
}

@Override
public int periodInSeconds() {
return 60;
Expand Down
43 changes: 32 additions & 11 deletions ...service/src/main/java/io/holoinsight/server/common/service/impl/AlertRuleServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,21 @@
import io.holoinsight.server.common.EventBusHolder;
import io.holoinsight.server.common.J;
import io.holoinsight.server.common.MD5Hash;
import io.holoinsight.server.common.MonitorPageRequest;
import io.holoinsight.server.common.MonitorPageResult;
import io.holoinsight.server.common.dao.converter.AlarmRuleConverter;
import io.holoinsight.server.common.dao.entity.AlarmBlock;
import io.holoinsight.server.common.dao.entity.AlarmRule;
import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO;
import io.holoinsight.server.common.dao.entity.dto.AlertRuleExtra;
import io.holoinsight.server.common.dao.entity.dto.alarm.AlarmRuleConf;
import io.holoinsight.server.common.dao.entity.dto.alarm.trigger.DataSource;
import io.holoinsight.server.common.dao.entity.dto.alarm.trigger.Filter;
import io.holoinsight.server.common.dao.entity.dto.alarm.trigger.Trigger;
import io.holoinsight.server.common.dao.mapper.AlarmRuleMapper;
import io.holoinsight.server.common.dao.entity.AlarmBlock;
import io.holoinsight.server.common.dao.entity.AlarmRule;
import io.holoinsight.server.common.dao.entity.dto.AlarmRuleDTO;
import io.holoinsight.server.common.dao.entity.dto.AlertRuleExtra;
import io.holoinsight.server.common.service.AlertBlockService;
import io.holoinsight.server.common.service.AlertRuleService;
import io.holoinsight.server.common.service.RequestContextAdapter;
import io.holoinsight.server.common.MonitorPageRequest;
import io.holoinsight.server.common.MonitorPageResult;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
Expand Down Expand Up @@ -57,29 +57,50 @@ public class AlertRuleServiceImpl extends ServiceImpl<AlarmRuleMapper, AlarmRule

@Override
public Long save(AlarmRuleDTO alarmRuleDTO) {
return save(alarmRuleDTO, true);
}

@Override
public Boolean updateById(AlarmRuleDTO alarmRuleDTO) {
return updateById(alarmRuleDTO, true);
}

@Override
public Boolean deleteById(Long id) {
return deleteById(id, true);
}

@Override
public Long save(AlarmRuleDTO alarmRuleDTO, boolean busPost) {
genMd5(alarmRuleDTO);
AlarmRule alarmRule = alarmRuleConverter.dtoToDO(alarmRuleDTO);
this.save(alarmRule);
EventBusHolder.post(alarmRuleConverter.doToDTO(alarmRule));
if (busPost) {
EventBusHolder.post(alarmRuleConverter.doToDTO(alarmRule));
}
return alarmRule.getId();
}

@Override
public Boolean updateById(AlarmRuleDTO alarmRuleDTO) {
public Boolean updateById(AlarmRuleDTO alarmRuleDTO, boolean busPost) {
genMd5(alarmRuleDTO);
AlarmRule alarmRule = alarmRuleConverter.dtoToDO(alarmRuleDTO);
EventBusHolder.post(alarmRuleDTO);
if (busPost) {
EventBusHolder.post(alarmRuleDTO);
}
return this.updateById(alarmRule);
}

@Override
public Boolean deleteById(Long id) {
public Boolean deleteById(Long id, boolean busPost) {
AlarmRule alarmRule = getById(id);
if (null == alarmRule) {
return true;
}
alarmRule.setStatus((byte) 0);
EventBusHolder.post(alarmRuleConverter.doToDTO(alarmRule));
if (busPost) {
EventBusHolder.post(alarmRuleConverter.doToDTO(alarmRule));
}
return this.removeById(id);
}

Expand Down
24 changes: 14 additions & 10 deletions ...va/io/holoinsight/server/home/web/security/custom/AlarmRuleLevelAuthorizationChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@
import io.holoinsight.server.common.dao.mapper.AlarmRuleMapper;
import io.holoinsight.server.common.dao.mapper.AlertTemplateMapper;
import io.holoinsight.server.common.service.RequestContextAdapter;
import io.holoinsight.server.home.dal.mapper.CustomPluginMapper;
import io.holoinsight.server.home.dal.model.CustomPlugin;
import io.holoinsight.server.common.service.SuperCacheService;
import io.holoinsight.server.home.web.common.ParaCheckUtil;
import io.holoinsight.server.home.web.security.LevelAuthorizationCheckResult;
import io.holoinsight.server.home.web.security.LevelAuthorizationMetaData;
Expand Down Expand Up @@ -71,6 +70,8 @@ public class AlarmRuleLevelAuthorizationChecker extends AbstractQueryChecker
private AlarmHistoryFacadeImplChecker historyFacadeImplChecker;
@Autowired
private ParameterSecurityService parameterSecurityService;
@Autowired
private SuperCacheService superCacheService;
private static final Pattern timePattern =
Pattern.compile("^(?:[01]\\d|2[0-3]):(?:[0-5]\\d):(?:[0-5]\\d)$");

Expand All @@ -92,12 +93,11 @@ public class AlarmRuleLevelAuthorizationChecker extends AbstractQueryChecker
new HashSet<>(Arrays.asList("default", "gradual", "fixed"));
private static final Set<String> aggregators = new HashSet<>(Arrays.asList("sum", "avg", "min",
"max", "count", "none", "SUM", "AVG", "MIN", "MAX", "COUNT", "NONE"));
private static final Set<String> metricTypes =
new HashSet<>(Arrays.asList("app", "cache", "log", "oss", "trace", "system", "metric",
"service", "function", "pg", "mongodb", "db", "miniProgram", "mysql"));
private static final Set<String> products = new HashSet<>(
Arrays.asList("JVM", "Function", "OceanBase", "Tbase", "PortCheck", "System", "MiniProgram",
"Spanner", "IoT", "APM", "Mysql", "SLB", "SOFAMQX", "Postgres", "Gateway"));
private static final Set<String> defaultMetricTypes =
new HashSet<>(Arrays.asList("message", "loadbalancing"));
// private static final Set<String> products = new HashSet<>(
// Arrays.asList("JVM", "Function", "OceanBase", "Tbase", "PortCheck", "System", "MiniProgram",
// "Spanner", "IoT", "APM", "Mysql", "SLB", "SOFAMQX", "Postgres", "Gateway"));

@Override
public LevelAuthorizationCheckResult check(LevelAuthorizationMetaData levelAuthMetaData,
Expand Down Expand Up @@ -423,12 +423,16 @@ private LevelAuthorizationCheckResult checkRule(Map<String, Object> ruleMap, Str

private LevelAuthorizationCheckResult checkDatasources(List<DataSource> datasources,
String tenant, String workspace) {
Set<String> metricTypes = this.superCacheService.getSc().metricTypes;
Set<String> products = this.superCacheService.getSc().integrationProducts;
for (DataSource dataSource : datasources) {
if (StringUtils.isNotEmpty(dataSource.getMetricType())
&& !metricTypes.contains(dataSource.getMetricType())) {
&& !CollectionUtils.isEmpty(metricTypes)
&& !metricTypes.contains(dataSource.getMetricType())
&& !defaultMetricTypes.contains(dataSource.getMetricType())) {
return failCheckResult("invalid metric type %s", dataSource.getMetricType());
}
if (StringUtils.isNotEmpty(dataSource.getProduct())
if (StringUtils.isNotEmpty(dataSource.getProduct()) && !CollectionUtils.isEmpty(products)
&& !products.contains(dataSource.getProduct())) {
return failCheckResult("invalid product %s", dataSource.getProduct());
}
Expand Down
51 changes: 28 additions & 23 deletions ...y/registry-core/src/main/java/io/holoinsight/server/registry/core/agent/AgentService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,6 @@
*/
package io.holoinsight.server.registry.core.agent;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.google.common.collect.Maps;
import com.xzchaoo.commons.basic.Ack;
import com.xzchaoo.commons.basic.Acks;
Expand All @@ -31,22 +11,40 @@
import com.xzchaoo.commons.batchprocessor.Flusher;
import com.xzchaoo.commons.stat.StatAccumulator;
import com.xzchaoo.commons.stat.StringsKey;

import io.holoinsight.server.common.JsonUtils;
import io.holoinsight.server.common.MetricsUtils;
import io.holoinsight.server.common.NetUtils;
import io.holoinsight.server.common.auth.AuthInfo;
import io.holoinsight.server.common.dao.entity.GaeaAgentDO;
import io.holoinsight.server.common.dao.entity.GaeaAgentDOExample;
import io.holoinsight.server.common.dao.mapper.GaeaAgentDOMapper;
import io.holoinsight.server.common.event.EventBusHolder;
import io.holoinsight.server.common.threadpool.CommonThreadPools;
import io.holoinsight.server.meta.common.model.QueryExample;
import io.holoinsight.server.meta.facade.model.MetaType;
import io.holoinsight.server.meta.facade.service.AgentHeartBeatService;
import io.holoinsight.server.meta.facade.service.DataClientService;
import io.holoinsight.server.common.event.EventBusHolder;
import io.holoinsight.server.common.MetricsUtils;
import io.holoinsight.server.registry.grpc.agent.AgentK8sInfo;
import io.holoinsight.server.registry.grpc.agent.RegisterAgentRequest;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
* <p>
Expand All @@ -60,6 +58,8 @@ public class AgentService {
private static final Logger LOGGER = LoggerFactory.getLogger("AGENT");
private static final StatAccumulator<StringsKey> AGENT_HEARTBEAT_STAT =
MetricsUtils.SM.create("agent.heartbeat");
private static final StatAccumulator<StringsKey> AGENTID_HEARTBEAT_STAT =
MetricsUtils.SM.create("agentId.heartbeat");

@Autowired
private GaeaAgentDOMapper mapper;
Expand Down Expand Up @@ -107,6 +107,11 @@ private void flushHeartbeats(List<String> agentIds) {
long end = System.currentTimeMillis();

AGENT_HEARTBEAT_STAT.add(StringsKey.EMPTY, new long[] {1, count, end - begin});
if (CollectionUtils.isNotEmpty(agentIds)) {
for (String agentId : agentIds) {
AGENTID_HEARTBEAT_STAT.add(StringsKey.of(agentId), new long[] {1});
}
}
}

@PreDestroy
Expand Down
Loading

0 comments on commit 73fba08

Please sign in to comment.