[Metricbeat] Add debug log for cloudwatch metricset (#18074)

* add debug log for cloudwatch metricset

x-pack/metricbeat/module/aws/aws.go

@@ -103,6 +103,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		// collecting the first one.
 		if output.AccountAliases != nil {
 			metricSet.AccountName = output.AccountAliases[0]
+			base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.AccountName)
 		}
 	}
 
@@ -115,6 +116,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		base.Logger().Warn("failed to get caller identity, please check permission setting: ", err)
 	} else {
 		metricSet.AccountID = *outputIdentity.Account
+		base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID)
 	}
 
 	// Construct MetricSet with a full regions list

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -18,6 +18,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/aws"
@@ -51,6 +52,7 @@ func init() {
 // interface methods except for Fetch.
 type MetricSet struct {
 	*aws.MetricSet
+	logger            *logp.Logger
 	CloudwatchConfigs []Config `config:"metrics" validate:"nonzero,required"`
 }
 
@@ -113,6 +115,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	return &MetricSet{
 		MetricSet:         metricSet,
+		logger:            logp.NewLogger(metricsetName),
 		CloudwatchConfigs: config.CloudwatchMetrics,
 	}, nil
 }
@@ -132,10 +135,13 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 
 	// Get listMetricDetailTotal and namespaceDetailTotal from configuration
 	listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig()
+	m.logger.Debugf("listMetricDetailTotal = %s", listMetricDetailTotal)
+	m.logger.Debugf("namespaceDetailTotal = %s", namespaceDetailTotal)
 
 	// Create events based on listMetricDetailTotal from configuration
 	if len(listMetricDetailTotal.metricsWithStats) != 0 {
 		for _, regionName := range m.MetricSet.RegionsList {
+			m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
 			awsConfig := m.MetricSet.AwsConfig.Copy()
 			awsConfig.Region = regionName
 
@@ -150,6 +156,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 				return errors.Wrap(err, "createEvents failed for region "+regionName)
 			}
 
+			m.logger.Debugf("Collected metrics of metrics = %d", len(eventsWithIdentifier))
+
 			err = reportEvents(eventsWithIdentifier, report)
 			if err != nil {
 				return errors.Wrap(err, "reportEvents failed")
@@ -158,6 +166,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 	}
 
 	for _, regionName := range m.MetricSet.RegionsList {
+		m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
 		awsConfig := m.MetricSet.AwsConfig.Copy()
 		awsConfig.Region = regionName
 
@@ -169,9 +178,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 
 		// Create events based on namespaceDetailTotal from configuration
 		for namespace, namespaceDetails := range namespaceDetailTotal {
+			m.logger.Debugf("Collected metrics from namespace %s", namespace)
+
 			listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch)
 			if err != nil {
-				m.Logger().Info(err.Error())
+				m.logger.Info(err.Error())
 				continue
 			}
 
@@ -189,6 +200,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 				return errors.Wrap(err, "createEvents failed for region "+regionName)
 			}
 
+			m.logger.Debugf("Collected number of metrics = %d", len(eventsWithIdentifier))
+
 			err = reportEvents(eventsWithIdentifier, report)
 			if err != nil {
 				return errors.Wrap(err, "reportEvents failed")
@@ -434,12 +447,14 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 
 	// Construct metricDataQueries
 	metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.Period)
+	m.logger.Debugf("Number of MetricDataQueries = %d", len(metricDataQueries))
 	if len(metricDataQueries) == 0 {
 		return events, nil
 	}
 
 	// Use metricDataQueries to make GetMetricData API calls
 	metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime)
+	m.logger.Debugf("Number of metricDataResults = %d", len(metricDataResults))
 	if err != nil {
 		return events, errors.Wrap(err, "GetMetricDataResults failed")
 	}
@@ -482,7 +497,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 		resourceTagMap, err := aws.GetResourcesTags(svcResourceAPI, []string{resourceType})
 		if err != nil {
 			// If GetResourcesTags failed, continue report event just without tags.
-			m.Logger().Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName))
+			m.logger.Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName))
 		}
 
 		if len(tagsFilter) != 0 && len(resourceTagMap) == 0 {

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
@@ -1232,6 +1233,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) {
 	m := MetricSet{}
 	m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}}
 	m.MetricSet = &aws.MetricSet{Period: 5}
+	m.logger = logp.NewLogger("test")
 
 	mockTaggingSvc := &MockResourceGroupsTaggingClient{}
 	mockCloudwatchSvc := &MockCloudWatchClient{}
@@ -1272,6 +1274,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) {
 	m := MetricSet{}
 	m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}}
 	m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID}
+	m.logger = logp.NewLogger("test")
 
 	mockTaggingSvc := &MockResourceGroupsTaggingClient{}
 	mockCloudwatchSvc := &MockCloudWatchClientWithoutDim{}