From b52c1af19dfc456275c9c552c3c711d10fff7528 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 24 Sep 2021 08:41:44 -0600 Subject: [PATCH 1/3] Remove deprecated AWS config options --- x-pack/libbeat/common/aws/credentials.go | 10 ------- .../docs/aws-credentials-config.asciidoc | 4 +-- x-pack/metricbeat/module/aws/aws.go | 14 ---------- .../module/aws/cloudwatch/cloudwatch.go | 28 ++----------------- .../module/aws/elb/_meta/docs.asciidoc | 1 - .../module/aws/mtest/integration.go | 1 - .../scripts/test_cloudwatch_metricset.yml | 1 - 7 files changed, 4 insertions(+), 55 deletions(-) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 4ca498e1aa52..deef1196916f 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -15,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -28,7 +27,6 @@ type ConfigAWS struct { SharedCredentialFile string `config:"shared_credential_file"` Endpoint string `config:"endpoint"` RoleArn string `config:"role_arn"` - AWSPartition string `config:"aws_partition"` // Deprecated. ProxyUrl *url.URL `config:"proxy_url"` } @@ -148,11 +146,3 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName } return awsConfig } - -// Validate checks for deprecated config option -func (c ConfigAWS) Validate() error { - if c.AWSPartition != "" { - cfgwarn.Deprecate("8.0.0", "aws_partition is deprecated. Please use endpoint instead.") - } - return nil -} diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index a1cefd40bf75..d7223720ad4d 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -231,9 +231,7 @@ After these steps are done, IAM role ARN can be used for authentication in Metri Temporary security credentials has a limited lifetime and consists of an access key ID, a secret access key, and a security token which typically returned from `GetSessionToken`. MFA-enabled IAM users would need to submit an MFA code -while calling `GetSessionToken`. `default_region` identifies the AWS Region -whose servers you want to send your first API request to by default. This is -typically the Region closest to you, but it can be any Region. Please see +while calling `GetSessionToken`. Please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details. `sts get-session-token` AWS CLI can be used to generate temporary credentials. For example. with MFA-enabled: diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 0b59f9ebc7fc..0e04ad6f4fa3 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -100,20 +100,6 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { base.Logger().Debug("Metricset level config for tags filter: ", metricSet.TagsFilter) base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset") - // Get IAM account name, set region by aws_partition, default is aws global partition - // refer https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html - if config.AWSConfig.AWSPartition != "" && config.AWSConfig.Endpoint != "" { - base.Logger().Warn("aws_partition is deprecated. Please use endpoint and regions instead.") - } - switch config.AWSConfig.AWSPartition { - case "aws-cn": - awsConfig.Region = "cn-north-1" - case "aws-us-gov": - awsConfig.Region = "us-gov-east-1" - default: - awsConfig.Region = "us-east-1" - } - // If regions in config is not empty, then overwrite the awsConfig.Region if len(config.Regions) > 0 { awsConfig.Region = config.Regions[0] diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 31df9f638d6b..e5d46dc2b629 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -17,7 +17,6 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" @@ -68,22 +67,8 @@ type Config struct { Namespace string `config:"namespace" validate:"nonzero,required"` MetricName []string `config:"name"` Dimensions []Dimension `config:"dimensions"` - ResourceTypeFilter string `config:"tags.resource_type_filter"` // Deprecated. ResourceType string `config:"resource_type"` Statistic []string `config:"statistic"` - Tags []aws.Tag `config:"tags"` // Deprecated. -} - -// Validate checks for deprecated config options -func (c Config) Validate() error { - if c.Tags != nil { - cfgwarn.Deprecate("8.0.0", "tags is deprecated. Use tags_filter instead") - } - - if c.ResourceTypeFilter != "" { - cfgwarn.Deprecate("8.0.0", "tags.resource_type_filter is deprecated. Use resource_type instead") - } - return nil } type metricsWithStatistics struct { @@ -318,13 +303,6 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n resourceTypesWithTags := map[string][]aws.Tag{} for _, config := range m.CloudwatchConfigs { - // If tags_filter on metricset level is given, overwrite tags in - // cloudwatch metrics with tags_filter. - tagsFilter := config.Tags - if m.MetricSet.TagsFilter != nil { - tagsFilter = m.MetricSet.TagsFilter - } - // If there is no statistic method specified, then use the default. if config.Statistic == nil { config.Statistic = defaultStatistics @@ -358,9 +336,9 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n if config.ResourceType != "" { if _, ok := resourceTypesWithTags[config.ResourceType]; ok { - resourceTypesWithTags[config.ResourceType] = tagsFilter + resourceTypesWithTags[config.ResourceType] = m.MetricSet.TagsFilter } else { - resourceTypesWithTags[config.ResourceType] = append(resourceTypesWithTags[config.ResourceType], tagsFilter...) + resourceTypesWithTags[config.ResourceType] = append(resourceTypesWithTags[config.ResourceType], m.MetricSet.TagsFilter...) } } continue @@ -368,7 +346,7 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n configPerNamespace := namespaceDetail{ names: config.MetricName, - tags: tagsFilter, + tags: m.MetricSet.TagsFilter, statistics: config.Statistic, resourceTypeFilter: config.ResourceType, dimensions: cloudwatchDimensions, diff --git a/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc index 8f242e8867aa..db39fe5a025c 100644 --- a/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc @@ -32,7 +32,6 @@ image::./images/metricbeat-aws-elb-overview.png[] access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' # This module uses the aws cloudwatch metricset, all # the options for this metricset are also available here. ---- diff --git a/x-pack/metricbeat/module/aws/mtest/integration.go b/x-pack/metricbeat/module/aws/mtest/integration.go index 243d783e3658..20c1ffee0ee3 100644 --- a/x-pack/metricbeat/module/aws/mtest/integration.go +++ b/x-pack/metricbeat/module/aws/mtest/integration.go @@ -34,7 +34,6 @@ func GetConfigForTest(t *testing.T, metricSetName string, period string) map[str "metricsets": []string{metricSetName}, "access_key_id": accessKeyID, "secret_access_key": secretAccessKey, - "default_region": defaultRegion, "latency": "5m", // You can specify which region to run test on by using regions variable // "regions": []string{"us-east-1"}, diff --git a/x-pack/metricbeat/module/aws/mtest/scripts/test_cloudwatch_metricset.yml b/x-pack/metricbeat/module/aws/mtest/scripts/test_cloudwatch_metricset.yml index 1df966dc922d..b1480c435696 100644 --- a/x-pack/metricbeat/module/aws/mtest/scripts/test_cloudwatch_metricset.yml +++ b/x-pack/metricbeat/module/aws/mtest/scripts/test_cloudwatch_metricset.yml @@ -9,7 +9,6 @@ access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' metrics: - namespace: AWS/EC2 resource_type_filter: ec2 From 8a34b45fa3f6fc72778b9de023d724e5b8038189 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 27 Sep 2021 08:19:49 -0600 Subject: [PATCH 2/3] run mage fmt --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index e5d46dc2b629..1e67fceac700 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -64,11 +64,11 @@ type Dimension struct { // Config holds a configuration specific for cloudwatch metricset. type Config struct { - Namespace string `config:"namespace" validate:"nonzero,required"` - MetricName []string `config:"name"` - Dimensions []Dimension `config:"dimensions"` - ResourceType string `config:"resource_type"` - Statistic []string `config:"statistic"` + Namespace string `config:"namespace" validate:"nonzero,required"` + MetricName []string `config:"name"` + Dimensions []Dimension `config:"dimensions"` + ResourceType string `config:"resource_type"` + Statistic []string `config:"statistic"` } type metricsWithStatistics struct { From efe3cb0b8c52b876ec1d9a0b603d08a5ac910e6f Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 1 Oct 2021 10:20:00 -0600 Subject: [PATCH 3/3] add changelog --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b3efac5cbbc1..67e2ecb9352d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -22,6 +22,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Skip add_kubernetes_metadata processor when kubernetes metadata are already present {pull}27689[27689] - Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull}28006[28006] - Remove deprecated fields from kubernetes module {pull}28046[28046] +- Remove deprecated config option aws_partition. {pull}28120[28120] *Auditbeat* @@ -110,6 +111,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804] - Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933] - Remove deprecated fields in Kafka module. {pull}27938[27938] +- Remove deprecated config option default_region from aws module. {pull}28120[28120] *Packetbeat*