Skip to content

Commit

Permalink
[Metricbeat] Rename tags to tags_filter for cloudwatch metricset (ela…
Browse files Browse the repository at this point in the history 
…stic#16733)

* Deprecate tags config parameter to use tags_filter instead for cloudwatch metricset
  • Loading branch information
@kaiyan-sheng
kaiyan-sheng authored May 20, 2020
1 parent 01bee98 commit 7e5e98c
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 43 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ field. You can revert this change by configuring tags for the module and omittin
*Journalbeat*

*Metricbeat*

- Deprecate tags config parameter in cloudwatch metricset. {pull}16733[16733]

*Packetbeat*

Expand Down
21 changes: 15 additions & 6 deletions x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/pkg/errors"

"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
"github.com/elastic/beats/v7/libbeat/logp"
"github.com/elastic/beats/v7/metricbeat/mb"
awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
Expand Down Expand Up @@ -69,7 +70,15 @@ type Config struct {
Dimensions []Dimension `config:"dimensions"`
ResourceTypeFilter string `config:"tags.resource_type_filter"`
Statistic []string `config:"statistic"`
Tags []aws.Tag `config:"tags"`
Tags []aws.Tag `config:"tags"` // Deprecated.
}

// Validate checks for deprecated config options
func (c Config) Validate() error {
if c.Tags != nil {
cfgwarn.Deprecate("8.0.0", "tags is deprecated. Use tags_filter instead")
}
return nil
}

type metricsWithStatistics struct {
Expand Down Expand Up @@ -292,8 +301,9 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n
for _, config := range m.CloudwatchConfigs {
// If tags_filter on metricset level is given, overwrite tags in
// cloudwatch metrics with tags_filter.
tagsFilter := config.Tags
if m.MetricSet.TagsFilter != nil {
config.Tags = m.MetricSet.TagsFilter
tagsFilter = m.MetricSet.TagsFilter
}

// If there is no statistic method specified, then use the default.
Expand Down Expand Up @@ -327,18 +337,17 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n

if config.ResourceTypeFilter != "" {
if _, ok := resourceTypesWithTags[config.ResourceTypeFilter]; ok {
resourceTypesWithTags[config.ResourceTypeFilter] = config.Tags

resourceTypesWithTags[config.ResourceTypeFilter] = tagsFilter
} else {
resourceTypesWithTags[config.ResourceTypeFilter] = append(resourceTypesWithTags[config.ResourceTypeFilter], config.Tags...)
resourceTypesWithTags[config.ResourceTypeFilter] = append(resourceTypesWithTags[config.ResourceTypeFilter], tagsFilter...)
}
}
continue
}

configPerNamespace := namespaceDetail{
names: config.MetricName,
tags: config.Tags,
tags: tagsFilter,
statistics: config.Statistic,
resourceTypeFilter: config.ResourceTypeFilter,
dimensions: cloudwatchDimensions,
Expand Down
118 changes: 82 additions & 36 deletions x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,53 @@ func TestReadCloudwatchConfig(t *testing.T) {
resourceTypeFilters: resourceTypeFiltersEC2RDS,
}

resourceTypeFiltersEC2RDSWithTag := map[string][]aws.Tag{}
resourceTypeFiltersEC2RDSWithTag["ec2:instance"] = []aws.Tag{
{
Key: "name",
Value: "test",
},
}
resourceTypeFiltersEC2RDSWithTag["rds"] = []aws.Tag{
{
Key: "name",
Value: "test",
},
}
expectedListMetricWithDetailEC2RDSWithTag := listMetricWithDetail{
metricsWithStats: []metricsWithStatistics{
{
cloudwatch.Metric{
Dimensions: []cloudwatch.Dimension{{
Name: awssdk.String("InstanceId"),
Value: awssdk.String("i-1"),
}},
MetricName: awssdk.String("CPUUtilization"),
Namespace: awssdk.String("AWS/EC2"),
},
[]string{"Average"},
nil,
},
{
cloudwatch.Metric{
Dimensions: []cloudwatch.Dimension{{
Name: awssdk.String("DBClusterIdentifier"),
Value: awssdk.String("test1-cluster"),
},
{
Name: awssdk.String("Role"),
Value: awssdk.String("READER"),
}},
MetricName: awssdk.String("CommitThroughput"),
Namespace: awssdk.String("AWS/RDS"),
},
[]string{"Average"},
nil,
},
},
resourceTypeFilters: resourceTypeFiltersEC2RDSWithTag,
}

expectedNamespaceDetailLambda := map[string][]namespaceDetail{}
expectedNamespaceDetailLambda["AWS/Lambda"] = []namespaceDetail{
{
Expand Down Expand Up @@ -269,7 +316,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
tags: []aws.Tag{
{
Key: "name",
Value: "test-ec2",
Value: "test",
},
},
},
Expand All @@ -282,7 +329,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
tags: []aws.Tag{
{
Key: "name",
Value: "test-elb1",
Value: "test",
},
},
},
Expand All @@ -293,7 +340,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
tags: []aws.Tag{
{
Key: "name",
Value: "test-elb2",
Value: "test",
},
},
},
Expand All @@ -303,6 +350,12 @@ func TestReadCloudwatchConfig(t *testing.T) {
expectedNamespaceDetailELBLambda["AWS/Lambda"] = []namespaceDetail{
{
statistics: defaultStatistics,
tags: []aws.Tag{
{
Key: "name",
Value: "test",
},
},
},
}
expectedNamespaceDetailELBLambda["AWS/ELB"] = []namespaceDetail{
Expand All @@ -313,7 +366,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
tags: []aws.Tag{
{
Key: "name",
Value: "test-elb1",
Value: "test",
},
},
},
Expand All @@ -324,7 +377,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
tags: []aws.Tag{
{
Key: "name",
Value: "test-elb2",
Value: "test",
},
},
},
Expand Down Expand Up @@ -377,6 +430,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
cases := []struct {
title string
cloudwatchMetricsConfig []Config
tagsFilter []aws.Tag
expectedListMetricDetailTotal listMetricWithDetail
expectedNamespaceDetailTotal map[string][]namespaceDetail
}{
Expand All @@ -396,6 +450,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
Statistic: []string{"Average"},
},
},
nil,
expectedListMetricWithDetailEC2,
map[string][]namespaceDetail{},
},
Expand All @@ -418,6 +473,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
Namespace: "AWS/S3",
},
},
nil,
expectedListMetricWithDetailEC2,
expectedNamespaceWithDetailS3,
},
Expand Down Expand Up @@ -456,6 +512,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
ResourceTypeFilter: "rds",
},
},
nil,
expectedListMetricWithDetailEC2RDS,
expectedNamespaceDetailLambda,
},
Expand All @@ -472,6 +529,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
ResourceTypeFilter: "s3",
},
},
nil,
listMetricWithDetail{
resourceTypeFilters: map[string][]aws.Tag{},
},
Expand All @@ -485,6 +543,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
ResourceTypeFilter: "ec2",
},
},
nil,
listMetricWithDetail{
resourceTypeFilters: map[string][]aws.Tag{},
},
Expand All @@ -500,6 +559,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
Statistic: []string{"Average", "Maximum"},
},
},
nil,
listMetricWithDetail{
resourceTypeFilters: map[string][]aws.Tag{},
},
Expand All @@ -513,6 +573,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
MetricName: []string{"MemoryUsed"},
},
},
nil,
listMetricWithDetail{
resourceTypeFilters: map[string][]aws.Tag{},
},
Expand All @@ -525,36 +586,24 @@ func TestReadCloudwatchConfig(t *testing.T) {
Namespace: "AWS/EC2",
MetricName: []string{"CPUUtilization"},
ResourceTypeFilter: resourceTypeEC2,
Tags: []aws.Tag{
{
Key: "name",
Value: "test-ec2",
},
},
},
{
Namespace: "AWS/ELB",
MetricName: []string{"BackendConnectionErrors", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX"},
Statistic: []string{"Sum"},
ResourceTypeFilter: "elasticloadbalancing",
Tags: []aws.Tag{
{
Key: "name",
Value: "test-elb1",
},
},
},
{
Namespace: "AWS/ELB",
MetricName: []string{"HealthyHostCount", "SurgeQueueLength", "UnHealthyHostCount"},
Statistic: []string{"Maximum"},
ResourceTypeFilter: "elasticloadbalancing",
Tags: []aws.Tag{
{
Key: "name",
Value: "test-elb2",
},
},
},
},
[]aws.Tag{
{
Key: "name",
Value: "test",
},
},
listMetricWithDetail{
Expand All @@ -570,24 +619,12 @@ func TestReadCloudwatchConfig(t *testing.T) {
MetricName: []string{"BackendConnectionErrors", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX"},
Statistic: []string{"Sum"},
ResourceTypeFilter: "elasticloadbalancing",
Tags: []aws.Tag{
{
Key: "name",
Value: "test-elb1",
},
},
},
{
Namespace: "AWS/ELB",
MetricName: []string{"HealthyHostCount", "SurgeQueueLength", "UnHealthyHostCount"},
Statistic: []string{"Maximum"},
ResourceTypeFilter: "elasticloadbalancing",
Tags: []aws.Tag{
{
Key: "name",
Value: "test-elb2",
},
},
},
{
Namespace: "AWS/Lambda",
Expand Down Expand Up @@ -621,7 +658,13 @@ func TestReadCloudwatchConfig(t *testing.T) {
ResourceTypeFilter: "rds",
},
},
expectedListMetricWithDetailEC2RDS,
[]aws.Tag{
{
Key: "name",
Value: "test",
},
},
expectedListMetricWithDetailEC2RDSWithTag,
expectedNamespaceDetailELBLambda,
},
{
Expand All @@ -639,6 +682,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
ResourceTypeFilter: "ec2:instance",
},
},
nil,
listMetricWithDetail{
resourceTypeFilters: map[string][]aws.Tag{},
},
Expand All @@ -660,6 +704,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
Statistic: []string{"Average"},
},
},
nil,
expectedListMetricsEC2WithDim,
map[string][]namespaceDetail{},
},
Expand All @@ -668,6 +713,7 @@ func TestReadCloudwatchConfig(t *testing.T) {
for _, c := range cases {
t.Run(c.title, func(t *testing.T) {
m.CloudwatchConfigs = c.cloudwatchMetricsConfig
m.MetricSet.TagsFilter = c.tagsFilter
listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig()
assert.Equal(t, c.expectedListMetricDetailTotal, listMetricDetailTotal)
assert.Equal(t, c.expectedNamespaceDetailTotal, namespaceDetailTotal)
Expand Down

0 comments on commit 7e5e98c

Please sign in to comment.