Merge pull request #51 from KunfengHe/br_release_sdk_v3

cache dms&agent metrics

README.md

@@ -11,8 +11,8 @@ $ git clone https://github.com/huaweicloud/cloudeye-exporter
 
 ## (Option) Building The Discovery with Exact steps on clean Ubuntu 16.04 
 ```
-$ wget https://dl.google.com/go/go1.16.3.linux-amd64.tar.gz
-$ sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz
+$ wget https://dl.google.com/go/go1.17.6.linux-amd64.tar.gz
+$ sudo tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz
 $ export PATH=$PATH:/usr/local/go/bin # You should put in your .profile or .bashrc
 $ go version # to verify it runs and version #
 

README_cn.md

@@ -64,8 +64,8 @@ Prometheus是用于展示大型测量数据的开源可视化工具，在工业
 # 参考命令：
 mkdir cloudeye-exporter
 cd cloudeye-exporter
-wget https://github.com/huaweicloud/cloudeye-exporter/releases/download/v2.0.1/cloudeye-exporter.v2.0.1.tar.gz
-tar -xzvf cloudeye-exporter.v2.0.1.tar.gz
+wget https://github.com/huaweicloud/cloudeye-exporter/releases/download/v2.0.4/cloudeye-exporter.v2.0.4.tar.gz
+tar -xzvf cloudeye-exporter.v2.0.4.tar.gz
 ```
 2. 编辑clouds.yml文件配置公有云信息
 ```

collector/apic.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"strings"
 	"time"
 
 	"github.com/huaweicloud/cloudeye-exporter/logs"
@@ -41,6 +42,7 @@ func (getter APICInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metri
 			}
 			resourceInfos[GetResourceKeyFromMetricInfo(metrics[0])] = info
 			buildApisInfo(*instance.Id, resourceInfos, &filterMetrics, info)
+			buildNodeInfo(*instance.Id, resourceInfos, &filterMetrics, info)
 		}
 		apicInfo.LabelInfo = resourceInfos
 		apicInfo.FilterMetrics = filterMetrics
@@ -75,6 +77,47 @@ func buildApisInfo(instanceId string, resourceInfos map[string]labelInfo, filter
 	}
 }
 
+func buildNodeInfo(instanceId string, resourceInfos map[string]labelInfo, filterMetrics *[]cesmodel.MetricInfoList, instanceInfo labelInfo) {
+	sysConfigMap := getMetricConfigMap("SYS.APIC")
+	apiMetricNames, ok := sysConfigMap["instance_id,node_ip"]
+	if !ok {
+		logs.Logger.Warnf("Metric config is empty of SYS.APIC, dim_metric_name is instance_id,node_ip")
+		return
+	}
+	instance, err := showDetailsOfInstanceV2(instanceId)
+	if err != nil {
+		logs.Logger.Errorf("Get all apis of apic instances: %s", err.Error())
+		return
+	}
+	nodeIps := make([]string, len(*instance.NodeIps.Livedata)+len(*instance.NodeIps.Shubao))
+	nodeIps = append(nodeIps, *instance.NodeIps.Livedata...)
+	nodeIps = append(nodeIps, *instance.NodeIps.Shubao...)
+
+	for _, nodeIP := range *instance.NodeIps.Livedata {
+		metrics := buildDimensionMetrics(apiMetricNames, "SYS.APIC",
+			[]cesmodel.MetricsDimension{{Name: "instance_id", Value: instanceId}, {Name: "node_ip", Value: strings.ReplaceAll(nodeIP, ".", "_")}})
+		*filterMetrics = append(*filterMetrics, metrics...)
+		resourceInfos[GetResourceKeyFromMetricInfo(metrics[0])] = getNodeInfo(nodeIP, "livedata", instanceInfo)
+	}
+
+	for _, nodeIP := range *instance.NodeIps.Shubao {
+		metrics := buildDimensionMetrics(apiMetricNames, "SYS.APIC",
+			[]cesmodel.MetricsDimension{{Name: "instance_id", Value: instanceId}, {Name: "node_ip", Value: strings.ReplaceAll(nodeIP, ".", "_")}})
+		*filterMetrics = append(*filterMetrics, metrics...)
+		resourceInfos[GetResourceKeyFromMetricInfo(metrics[0])] = getNodeInfo(nodeIP, "shubao", instanceInfo)
+	}
+}
+
+func getNodeInfo(nodeIP, nodeType string, instanceInfo labelInfo) labelInfo {
+	appInfo := labelInfo{
+		Name:  []string{"nodeIP", "nodeType"},
+		Value: []string{nodeIP, nodeType},
+	}
+	appInfo.Name = append(appInfo.Name, instanceInfo.Name...)
+	appInfo.Value = append(appInfo.Value, instanceInfo.Value...)
+	return appInfo
+}
+
 func getAllAPICInstances() ([]model.RespInstanceBase, error) {
 	limit := int32(500)
 	offset := int64(0)
@@ -117,6 +160,16 @@ func getApisOfInstances(instanceID string) ([]model.ApiInfoPerPage, error) {
 	return apis, nil
 }
 
+func showDetailsOfInstanceV2(instanceID string) (*model.ShowDetailsOfInstanceV2Response, error) {
+	request := &model.ShowDetailsOfInstanceV2Request{InstanceId: instanceID}
+	response, err := getAPICSClient().ShowDetailsOfInstanceV2(request)
+	if err != nil {
+		logs.Logger.Errorf("Failed to get instance info[%s], error: %s", instanceID, err.Error())
+		return nil, err
+	}
+	return response, nil
+}
+
 func getAPICSClient() *apig.ApigClient {
 	return apig.NewApigClient(apig.ApigClientBuilder().WithCredential(
 		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).

collector/collector.go

@@ -69,6 +69,11 @@ func (exporter *BaseHuaweiCloudExporter) listMetrics(namespace string) ([]model.
 
 func (exporter *BaseHuaweiCloudExporter) setProData(ctx context.Context, ch chan<- prometheus.Metric,
 	dataList []model.BatchMetricData, allResourcesInfo map[string]labelInfo) {
+	defer func() {
+		if err := recover(); err != nil {
+			logs.Logger.Errorf("[%s] SetProData error: %+v", exporter.txnKey, err)
+		}
+	}()
 	for _, metric := range dataList {
 		exporter.debugMetricInfo(metric)
 		data, err := getLatestData(metric.Datapoints)
@@ -154,8 +159,13 @@ func (exporter *BaseHuaweiCloudExporter) collectMetricByNamespace(ctx context.Co
 	var wg sync.WaitGroup
 	count := 0
 	tmpMetrics := make([]model.MetricInfo, 0, exporter.ScrapeBatchSize)
-
+	metricsMap := make(map[string]bool, 0)
 	for _, metric := range allMetrics {
+		dimsValueKey := fmt.Sprintf("%s,%s", getDimsValueKey(metric.Dimensions), metric.MetricName)
+		if _, ok := metricsMap[dimsValueKey]; ok {
+			continue
+		}
+		metricsMap[dimsValueKey] = true
 		count++
 		tmpMetrics = append(tmpMetrics, transMetric(metric))
 		if (len(tmpMetrics) == exporter.ScrapeBatchSize) || (count == len(allMetrics)) {

collector/dms.go

@@ -23,26 +23,44 @@ var dmsInfo serversInfo
 type DMSInfo struct{}
 
 func (getter DMSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInfoList) {
-	resourceInfos := map[string]labelInfo{}
 	dmsInfo.Lock()
 	defer dmsInfo.Unlock()
-	if dmsInfo.LabelInfo == nil || time.Now().Unix() > dmsInfo.TTL {
-		for _, instance := range getDMSInstanceFromRMS() {
-			info := labelInfo{
-				Name:  []string{"instanceName", "epId"},
-				Value: []string{instance.Name, instance.EpId},
-			}
-			keys, values := getTags(instance.Tags)
-			info.Name = append(info.Name, keys...)
-			info.Value = append(info.Value, values...)
-			resourceInfos[instance.ID] = info
-		}
-		dmsInfo.LabelInfo = resourceInfos
+	if dmsInfo.LabelInfo == nil {
+		dmsInfo.LabelInfo, dmsInfo.FilterMetrics = getDMSResourceAndMetrics()
 		dmsInfo.TTL = time.Now().Add(TTL).Unix()
 	}
+	if time.Now().Unix() > dmsInfo.TTL {
+		go func() {
+			label, metrics := getDMSResourceAndMetrics()
+			dmsInfo.Lock()
+			defer dmsInfo.Unlock()
+			dmsInfo.LabelInfo = label
+			dmsInfo.FilterMetrics = metrics
+			dmsInfo.TTL = time.Now().Add(TTL).Unix()
+		}()
+	}
 	return dmsInfo.LabelInfo, dmsInfo.FilterMetrics
 }
 
+func getDMSResourceAndMetrics() (map[string]labelInfo, []model.MetricInfoList) {
+	resourceInfos := map[string]labelInfo{}
+	for _, instance := range getDMSInstanceFromRMS() {
+		info := labelInfo{
+			Name:  []string{"instanceName", "epId"},
+			Value: []string{instance.Name, instance.EpId},
+		}
+		keys, values := getTags(instance.Tags)
+		info.Name = append(info.Name, keys...)
+		info.Value = append(info.Value, values...)
+		resourceInfos[instance.ID] = info
+	}
+	allMetrics, err := listAllMetrics("SYS.DMS")
+	if err != nil {
+		logs.Logger.Errorf("[%s] Get all metrics of SYS.DMS error: %s", err.Error())
+	}
+	return resourceInfos, allMetrics
+}
+
 func getDMSInstanceFromRMS() []ResourceBaseInfo {
 	var resources []rmsmodel.ResourceEntity
 	kafkaResp, err := listResources("dms", "kafkas")

collector/ecs.go

@@ -13,7 +13,10 @@ import (
 	"github.com/huaweicloud/cloudeye-exporter/logs"
 )
 
-var ecsInfo serversInfo
+var (
+	ecsInfo    serversInfo
+	agtEcsInfo serversInfo
+)
 
 type ECSInfo struct{}
 
@@ -156,7 +159,30 @@ func getIPInfoFromProperties(properties *EcsProperties) string {
 type AGTECSInfo struct{}
 
 func (getter AGTECSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInfoList) {
-	ecsInfo.Lock()
-	defer ecsInfo.Unlock()
-	return ecsInfo.LabelInfo, nil
+	agtEcsInfo.Lock()
+	defer agtEcsInfo.Unlock()
+	if agtEcsInfo.LabelInfo == nil {
+		agtEcsInfo.FilterMetrics = getECSAGTMetrics()
+		agtEcsInfo.LabelInfo = ecsInfo.LabelInfo
+		agtEcsInfo.TTL = time.Now().Add(TTL).Unix()
+	}
+	if time.Now().Unix() > agtEcsInfo.TTL {
+		go func() {
+			metrics := getECSAGTMetrics()
+			agtEcsInfo.Lock()
+			defer agtEcsInfo.Unlock()
+			agtEcsInfo.FilterMetrics = metrics
+			agtEcsInfo.LabelInfo = ecsInfo.LabelInfo
+			agtEcsInfo.TTL = time.Now().Add(TTL).Unix()
+		}()
+	}
+	return agtEcsInfo.LabelInfo, agtEcsInfo.FilterMetrics
+}
+
+func getECSAGTMetrics() []model.MetricInfoList {
+	allMetrics, err := listAllMetrics("AGT.ECS")
+	if err != nil {
+		logs.Logger.Errorf("[%s] Get all metrics of AGT.ECS error: %s", err.Error())
+	}
+	return allMetrics
 }

collector/metrics.go

@@ -1,6 +1,8 @@
 package collector
 
 import (
+	"sync"
+
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config"
 	ces "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1"
@@ -12,8 +14,8 @@ import (
 )
 
 var (
-	host string
-	agentDimensions = make(map[string]string, 0)
+	host            string
+	agentDimensions = sync.Map{}
 )
 
 func getCESClient() *ces.CesClient {
@@ -75,11 +77,17 @@ func getCESClientV2() *cesv2.CesClient {
 }
 
 func getAgentOriginValue(value string) string {
-	originValue, ok := agentDimensions[value]
-	if ok {
-		return originValue
+	originValue, ok := agentDimensions.Load(value)
+	if !ok {
+		return value
+	}
+
+	originV, ok := originValue.(string)
+	if !ok {
+		return value
 	}
-	return value
+
+	return originV
 }
 
 func loadAgentDimensions(instanceID string) {
@@ -98,7 +106,7 @@ func loadAgentDimensions(instanceID string) {
 			return
 		}
 		for _, dimension := range *response.Dimensions {
-			agentDimensions[*dimension.Value] = *dimension.OriginValue
+			agentDimensions.Store(*dimension.Value, *dimension.OriginValue)
 		}
 	}
 }

collector/utils.go

@@ -151,6 +151,14 @@ func getDimsNameKey(dims []model.MetricsDimension) string {
 	return strings.Join(dimsNamesList, ",")
 }
 
+func getDimsValueKey(dims []model.MetricsDimension) string {
+	dimsValuesList := make([]string, 0, len(dims))
+	for _, dim := range dims {
+		dimsValuesList = append(dimsValuesList, dim.Value)
+	}
+	return strings.Join(dimsValuesList, ",")
+}
+
 func buildSingleDimensionMetrics(metricNames []string, namespace, dimName, dimValue string) []model.MetricInfoList {
 	filterMetrics := make([]model.MetricInfoList, len(metricNames))
 	for index := range metricNames {

go.mod

@@ -3,9 +3,10 @@ module github.com/huaweicloud/cloudeye-exporter
 go 1.16
 
 require (
-	github.com/cihub/seelog v0.0.0-20191126193922-f561c5e57575
 	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.35
 	github.com/prometheus/client_golang v1.12.2
+	go.uber.org/zap v1.24.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v3 v3.0.1
 )