From 4b9679a46c6a937c31720ff03e0adf000366666c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 8 Aug 2019 12:05:31 -0600 Subject: [PATCH 1/4] Add calculate_rate for NetworkIn/Out, DiskRead/WriteBytes/Ops --- metricbeat/docs/fields.asciidoc | 60 +++++++++++++++++ x-pack/metricbeat/module/aws/_meta/config.yml | 1 + .../metricbeat/module/aws/ec2/_meta/data.json | 48 +++++++------ .../module/aws/ec2/_meta/docs.asciidoc | 15 +++++ .../module/aws/ec2/_meta/fields.yml | 24 +++++++ x-pack/metricbeat/module/aws/ec2/ec2.go | 67 ++++++++++++++++++- .../module/aws/ec2/ec2_integration_test.go | 1 + x-pack/metricbeat/module/aws/fields.go | 2 +- .../module/aws/mtest/integration.go | 1 + x-pack/metricbeat/modules.d/aws.yml.disabled | 1 + 10 files changed, 199 insertions(+), 21 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 41163f34a24d..fcd3fea2fa9b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -976,6 +976,16 @@ format: bytes -- +*`aws.ec2.network.in.bytes_per_sec`*:: ++ +-- +Network received rate bytes per second, calculated with dividing network.in.bytes by 300 or by 60(detailed monitoring). + + +type: long + +-- + *`aws.ec2.network.out.bytes`*:: + -- @@ -988,6 +998,16 @@ format: bytes -- +*`aws.ec2.network.out.bytes_per_sec`*:: ++ +-- +Network sent out rate bytes per second, calculated with dividing network.out.bytes by 300 or by 60(detailed monitoring). + + +type: long + +-- + *`aws.ec2.diskio.read.bytes`*:: + -- @@ -1000,6 +1020,16 @@ format: bytes -- +*`aws.ec2.diskio.read.bytes_per_sec`*:: ++ +-- +Instance store volumes read rate bytes per second, calculated with dividing diskio.read.bytes by 300 or by 60(detailed monitoring). + + +type: long + +-- + *`aws.ec2.diskio.write.bytes`*:: + -- @@ -1012,12 +1042,32 @@ format: bytes -- +*`aws.ec2.diskio.write.bytes_per_sec`*:: ++ +-- +Instance store volumes write rate bytes per second, calculated with dividing diskio.write.bytes by 300 or by 60(detailed monitoring). + + +type: long + +-- + *`aws.ec2.diskio.read.ops`*:: + -- Completed read operations from all instance store volumes available to the instance in a specified period of time. +type: long + +-- + +*`aws.ec2.diskio.read.ops_per_sec`*:: ++ +-- +Read operations count per second, calculated with dividing diskio.read.ops by 300 or by 60(detailed monitoring). + + type: long -- @@ -1028,6 +1078,16 @@ type: long Completed write operations to all instance store volumes available to the instance in a specified period of time. +type: long + +-- + +*`aws.ec2.diskio.write.ops_per_sec`*:: ++ +-- +Write operations count per second, calculated with dividing diskio.write.ops by 300 or by 60(detailed monitoring). + + type: long -- diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 90dc6dc1af3b..f4bd6405ac8a 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -4,6 +4,7 @@ - ec2 regions: - us-east-1 + calculate_rate: false - module: aws period: 300s metricsets: diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/data.json b/x-pack/metricbeat/module/aws/ec2/_meta/data.json index 0ad102cc4ac9..455401a90709 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/data.json +++ b/x-pack/metricbeat/module/aws/ec2/_meta/data.json @@ -3,41 +3,45 @@ "aws": { "ec2": { "cpu": { - "credit_balance": 144, - "credit_usage": 0.005335, + "credit_balance": 576, + "credit_usage": 0.194305, "surplus_credit_balance": 0, "surplus_credits_charged": 0, "total": { - "pct": 0.0989071038248902 + "pct": 2.0499999999883576 } }, "diskio": { "read": { "bytes": 0, - "count": 0 + "bytes_per_sec": 0, + "count": 0, + "count_per_sec": 0 }, "write": { "bytes": 0, - "count": 0 + "bytes_per_sec": 0, + "count": 0, + "count_per_sec": 0 } }, "instance": { "core": { - "count": 1 + "count": 2 }, "image": { - "id": "ami-05b3bcf7f311194b3" + "id": "ami-f920cd94" }, "monitoring": { "state": "disabled" }, "private": { - "dns_name": "ip-172-31-10-23.ap-southeast-1.compute.internal", - "ip": "172.31.10.23" + "dns_name": "ip-10-0-0-148.ec2.internal", + "ip": "10.0.0.148" }, "public": { - "dns_name": "ec2-18-136-198-93.ap-southeast-1.compute.amazonaws.com", - "ip": "18.136.198.93" + "dns_name": "ec2-54-226-109-162.compute-1.amazonaws.com", + "ip": "54.226.109.162" }, "state": { "code": 16, @@ -47,12 +51,14 @@ }, "network": { "in": { - "bytes": 85.2, - "packets": 1.4 + "bytes": 1297480, + "bytes_per_sec": 4324.933333333333, + "packets": 2734.8 }, "out": { - "bytes": 108.8, - "packets": 2 + "bytes": 395706.4, + "bytes_per_sec": 1319.0213333333334, + "packets": 2672.8 } }, "status": { @@ -60,18 +66,22 @@ "check_failed_instance": 0, "check_failed_system": 0 } + }, + "tags": { + "Cost": "$614.01", + "Name": "container-registry-test-ui.elastic.co" } }, "cloud": { - "availability_zone": "ap-southeast-1b", + "availability_zone": "us-east-1b", "instance": { - "id": "i-0c68eeb552231a8d0" + "id": "i-77f84332" }, "machine": { - "type": "t2.micro" + "type": "t2.medium" }, "provider": "aws", - "region": "ap-southeast-1" + "region": "us-east-1" }, "event": { "dataset": "aws.ec2", diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc index 5428dfb80446..3e892db4f8af 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc @@ -47,6 +47,20 @@ The aws ec2 metricset comes with a predefined dashboard. For example: image::./images/metricbeat-aws-ec2-overview.png[] +[float] +=== Metricset-specific configuration notes +* *calculate_rate*: If calculate_rate is true, 6 rate metrics will be calculated: +|=== +|Rate Metric Name | Equation | Equation with Detailed Monitoring +|network.in.bytes_per_sec | network.in.bytes / 300 | network.in.bytes / 60 +|network.out.bytes_per_sec | network.out.bytes / 300 | network.out.bytes / 60 +|diskio.read.bytes_per_sec | diskio.read.bytes / 300 | diskio.read.bytes / 60 +|diskio.write.bytes_per_sec | diskio.write.bytes / 300 | diskio.write.bytes / 60 +|diskio.read.ops_per_sec | diskio.read.ops / 300 | diskio.read.ops / 60 +|diskio.write.ops_per_sec | diskio.write.ops / 300 | diskio.write.ops / 60 +|=== +Default calculate_rate is false. + [float] === Configuration example [source,yaml] @@ -58,4 +72,5 @@ image::./images/metricbeat-aws-ec2-overview.png[] access_key_id: '' secret_access_key: '' session_token: '' + calculate_rate: false ---- diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml b/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml index 34e6686d24fa..c856c980b17e 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml @@ -37,29 +37,53 @@ format: bytes description: > The number of bytes received on all network interfaces by the instance. + - name: network.in.bytes_per_sec + type: long + description: > + Network received rate bytes per second, calculated with dividing network.in.bytes by 300 or by 60(detailed monitoring). - name: network.out.bytes type: long format: bytes description: > The number of bytes sent out on all network interfaces by the instance. + - name: network.out.bytes_per_sec + type: long + description: > + Network sent out rate bytes per second, calculated with dividing network.out.bytes by 300 or by 60(detailed monitoring). - name: diskio.read.bytes type: long format: bytes description: > Bytes read from all instance store volumes available to the instance. + - name: diskio.read.bytes_per_sec + type: long + description: > + Instance store volumes read rate bytes per second, calculated with dividing diskio.read.bytes by 300 or by 60(detailed monitoring). - name: diskio.write.bytes type: long format: bytes description: > Bytes written to all instance store volumes available to the instance. + - name: diskio.write.bytes_per_sec + type: long + description: > + Instance store volumes write rate bytes per second, calculated with dividing diskio.write.bytes by 300 or by 60(detailed monitoring). - name: diskio.read.ops type: long description: > Completed read operations from all instance store volumes available to the instance in a specified period of time. + - name: diskio.read.ops_per_sec + type: long + description: > + Read operations count per second, calculated with dividing diskio.read.ops by 300 or by 60(detailed monitoring). - name: diskio.write.ops type: long description: > Completed write operations to all instance store volumes available to the instance in a specified period of time. + - name: diskio.write.ops_per_sec + type: long + description: > + Write operations count per second, calculated with dividing diskio.write.ops by 300 or by 60(detailed monitoring). - name: status.check_failed type: long description: > diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 7148b03cd87a..9a21aabe3bc4 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/elastic/beats/libbeat/common" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" @@ -42,6 +44,7 @@ func init() { // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet + CalculateRate bool `config:"calculate_rate"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -62,8 +65,18 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { base.Logger().Info(err) } + config := struct { + CalculateRate bool `config:"calculate_rate"` + }{} + + err = base.Module().UnpackConfig(&config) + if err != nil { + return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") + } + return &MetricSet{ - MetricSet: metricSet, + MetricSet: metricSet, + CalculateRate: config.CalculateRate, }, nil } @@ -160,6 +173,9 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met metricSetFieldResults[instanceID] = map[string]interface{}{} } + // monitoring state for each instance + monitoringStates := map[string]string{} + // Find a timestamp for all metrics in output timestamp := aws.FindTimestamp(getMetricDataResults) if !timestamp.IsZero() { @@ -193,6 +209,8 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met return events, errors.Wrap(err, "instance.Monitoring.State.MarshalValue failed") } + monitoringStates[instanceID] = monitoringState + events[instanceID].MetricSetFields.Put("instance.image.id", *instanceOutput[instanceID].ImageId) events[instanceID].MetricSetFields.Put("instance.state.name", instanceStateName) events[instanceID].MetricSetFields.Put("instance.state.code", *instanceOutput[instanceID].State.Code) @@ -227,6 +245,10 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met return events, errors.Wrap(err, "EventMapping failed") } + if m.CalculateRate { + calculateRate(resultMetricsetFields, monitoringStates[instanceID]) + } + events[instanceID].MetricSetFields.Update(resultMetricsetFields) if len(events[instanceID].MetricSetFields) < 5 { m.Logger().Info("Missing Cloudwatch data, this is expected for non-running instances" + @@ -239,6 +261,49 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met return events, nil } +func calculateRate(resultMetricsetFields common.MapStr, monitoringState string) { + var period = 300.0 + if monitoringState != "disabled" { + period = 60.0 + } + + networkIn, err := resultMetricsetFields.GetValue("network.in.bytes") + if err == nil && networkIn != nil { + networkInRate := networkIn.(float64) / period + resultMetricsetFields.Put("network.in.bytes_per_sec", networkInRate) + } + + networkOut, err := resultMetricsetFields.GetValue("network.out.bytes") + if err == nil && networkOut != nil { + networkOutRate := networkOut.(float64) / period + resultMetricsetFields.Put("network.out.bytes_per_sec", networkOutRate) + } + + diskReadBytes, err := resultMetricsetFields.GetValue("diskio.read.bytes") + if err == nil && networkOut != nil { + diskReadBytesRate := diskReadBytes.(float64) / period + resultMetricsetFields.Put("diskio.read.bytes_per_sec", diskReadBytesRate) + } + + diskWriteBytes, err := resultMetricsetFields.GetValue("diskio.write.bytes") + if err == nil && networkOut != nil { + diskWriteBytesRate := diskWriteBytes.(float64) / period + resultMetricsetFields.Put("diskio.write.bytes_per_sec", diskWriteBytesRate) + } + + diskReadOps, err := resultMetricsetFields.GetValue("diskio.read.count") + if err == nil && networkOut != nil { + diskReadOpsRate := diskReadOps.(float64) / period + resultMetricsetFields.Put("diskio.read.count_per_sec", diskReadOpsRate) + } + + diskWriteOps, err := resultMetricsetFields.GetValue("diskio.write.count") + if err == nil && networkOut != nil { + diskWriteOpsRate := diskWriteOps.(float64) / period + resultMetricsetFields.Put("diskio.write.count_per_sec", diskWriteOpsRate) + } +} + func getInstancesPerRegion(svc ec2iface.ClientAPI) (instanceIDs []string, instancesOutputs map[string]ec2.Instance, err error) { instancesOutputs = map[string]ec2.Instance{} output := ec2.DescribeInstancesOutput{NextToken: nil} diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go index acdf5a901f38..cf990b5dd738 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go @@ -67,6 +67,7 @@ func TestFetch(t *testing.T) { func TestData(t *testing.T) { config, info := mtest.GetConfigForTest("ec2", "300s") + config["calculate_rate"] = true if info != "" { t.Skip("Skipping TestData: " + info) } diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index c06911ea8005..b0c714e253f6 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsXE9z27YSv+dT7OSUdBL1tWnfIYc349ieV884iRO50yMDASsKNQjQ+GNZmX74N/hDipIoS5RIuW+mObQJKWF/v93FYncB6C3c4eI9kLl5AWC5FfgeXpK5efkCgKGhmpeWK/ke/vMCAOAbmZtvUCjmBAJVQiC1Bs7+GEOhJLdKc5lDgVZzamCqVRHenQvl2JxYOhu9ANAokBh8Dzl5ATDlKJh5H0Z/C5IUWKHxf+yi9B/UypXpSQuo1UGaA1mSm9EP9eNqPDX5E6ltPI4Psvj2DhdzpVn766wgZcllnj778oeXjc+1Yot/bknuB4YHIhxCSbhO+iFzAxqNcpqiGW0woLXqNlg0tbJD+rflMN+AKmkJlwbsDGtb2RmxMEeNYKgmJbI16/3hvwvzGaez5QAtNjcoLUwWwPh0itr/w/MwJVlh13CCCVrSeL5uyaYu6pFW3lbq2DTbDqUEs8xwOSyYEimfcmQwn6GEe4d60bAAkJKPWpEl9qMfVpwNtjscrDsdU24i1ont63d7EP3YtLNG67SsbHy+QjCybqfJeIHScCXNwTzbrdQj0QabaJUm6o3phRNz1LzCiTnRhPJfvPwwPmQO1WTpz8eRpT+fkuz5z61k833DBS3dyCpLxKjc8MlI3lAikGVTocj6B/aIGyVqitKSHEFNgQihKLHIPHCgqiidRXCS26QeohGo0z4migVwCc4gKBn0yKWxRFJsn3aeCNXIuM2cIXl7+BNK5gfEPldMUHv85ze/QxRifByMdmhig6nS4VPOcsG/Ez/sTrwTIvx3B0GMJMSwJvCoaLnEPCMGCKXaIQPD/RNuYU4MCOIknSEDpcFYoi2y7WSM06VwJjsBqSRqldGMPCBMEOXSMkSCk4IX3HtcTTcsW/5r5ze/n4cRPkSsKe/gBr6jVvsyNRmdEZ3jerjuiWrg0krYzxWprM+TGDA1l57ypr3fAJEshRU7cwa4pE573RDGuEdBBEQK7ZQl2rnSdyMuRyWhd2jNIEzT2KCRIn/wTid9vKjEA5cW9dSnSOuT7mnYytmT4A5hWTnbF24uR5OFxb1BT5UuiH0PbV/qRCgMMIgZnpNPX+Zh3NxxNdJI2Mn4fEgWISll8PjrcGas0ggPSrgCDZAHwgWZCASr9mcz19ziiel4mRalx9k7n2AdVfY14c9VUQr0i0iwgSpRh5XdHG4On9uQRi1VouaKeXe1vNjHWEOwCyM36R1om0PIGUusMyM6Q3qXTQkXvS2oX7FU2hq/7tsZ6lWkPvcpiTHIYKLsbPVlxAQBU1hF/VuzMBaL1Xc8ZqiCGAsFl87uTzKL452Y6xBEKjnPQKXdYvuSqUMKVdr/x8n2SsgvFTnqowsIpdGEQmF3PKvf8oLkOOLtc+Lgrs7VRZiUHoYf31dbzE/mmPh3wbesU0feBj12n64k475mXHoCQxs8rlkccwMofSzaUqHUQEvNH4jFEZMm86/6VWgaHS4+jYPgSr0b2caeKHnZ7onrjztAu7p5+MWn/hqNAWKMojxU5HOewl9nrG4iOB1KoWHwDX3u6ZUJWo9arBSXcFz64MIpXN3Ub155Bb+GiXKSVQtjV5WGKTSiirVr8+BAFMZd1+Eb8PU//PTvtxNuwUnDcxnq5SBkL6T9270VKbwqUTI/3f8C7aSMfzMzZy2X+dtQA/8FFnXBZfDpv3zGEjql1V+Rvd7ByM58kmeyEnXmQ/VQS0GS45Ojello6b2KyXHtSDE5ZTvy+sMxvVfNjms0a3bKRvPXi4MazdBX97WqwFKL9djebNeu5T9d1lN3WRmxZEIMZlRJiTSUaIPQqQRBQ1Bqxm9BNqmz/xHRsr+l4Kwg35WEr2nvFz6RAl+dff30OrgAEjrzQWM3KCqIadfVQbDOmzGmmZdUWxq+WCywUHoBlJSEcruAgKH64MWHXZ2MBnrOUFpfSq8vOH1QIN6s+q1xZSl8sV4bfyl1BLczbhoPfLrtWTjJ7x2GPfvg7/Un/LCdKMbCrT9649R8iDjTllIzq+CmZrq925LdO3SYMSztbJCZppz1Cgg5zdVnA698TvBj7MNovHdorHkNc8J9ghPaMZT6JNOT8gDboVedhXuRGdQPqDOSo7TZn2oyTMCIAmH85RrGQSCceYHgBQJzYQndqxSfakRfxWVx8py0Y0wKX/mHbcq6saWJZKqotJ5AbUWeGau0r9OfG3bCAeHExpbTGOSRF67IfMmfWU2kISHQZ5z16SNJDDQkwNVFtWNn4oadxzCCsxCAQtPxRhmbaxx/uW4HrwRDYzONpeA0LP+ZEcpmguSjYtIjfEHy3Duv4d/rGJ+k1u9CnqmMDf0m1EWI8X+cXYf4Uu+jdOLno0DG1UiVsQ4xSFtJHbgpTx4wuEdjxefmLja3r3783OwA++rEIFVyS06SlBE0vw3lEz7PXBR0jNtbXiAQ+OrRf022aaw93k7ez2a8auDGVKK5PDVt83Ex/nL9Bj4SzcnFh7h7urTXipgtiYeZkzKmx88UCDyAOPdjRy8doFhhHFb0dPiHm7CRvIwfPrlaBvN2ls2YIVRushwltlrzmAkYHLNBxVcCjVDiBXeaWWFpPf3Uiit6x7l171Dz/d3nIHRJBuAjUmfjXs2ToBgSJhS9GxZWLaVq4tdZ6S58cXMqLGvPNfvS4lv560I5DWdOK70Sl954akHYEwmFUJSIZ04rKj6r4cFiUSpN9AKsf2ZCnPTTcRcvoXIuw56R0wM7d0pLg0Qg1kO2u6ednWnl8lnp7IiqouDtjZm+MEYRXcJCAx9DgVv2WvqbiV5EHSe6gGNiUGQXF9d1jdQJVzEsLi4NamvegCsZsZgOYEU9dgIaBzoB1kOsm7Y0+kRXR5xqu2QpLm7O12eS4rELn8/5XKDg1sZ9Syo4SmviQTg6WzmQ4FeRFIVDiudjMRXOWNTLkHWABrKEqkdNcElV4UuLV1/j2K+XKtFkOuW0JaXzJKhwoZUQtEWdsapAvVw7qy97zVWdtYtx/TgsWD62Nxrf/qONMmtvpVSG6VErytlcBa3cpsH/f9TiS6sBJvJaEWTJHcrgwiHpXEk3d0I0KDavJfQbbaKIQ6JNDKWDgosiDgEXkvtBsU3WT8wF++6CKIhFSRcds5g+S/OEIMyejUQnRN2CC8EjiW35YmLRIZ0YikJo6DCccslj9Ulk7rylXl1cXL+uk5GuxPbPR4Yi9mTK0pFOt6xlWEbVbO5IoUu07oFAX8G8gt8tkg9lgdVg39EC3eL9UBRWl4SOFDqtCn8jLwr7Xql7yUNr75n6Do3Wn6LUlTy2UyZcEr0IrYYq3SuIz+I3u7ixo62fbNY26K5vJ/S7ldDSyWwIBC8Qplxgt35mA/56Q3Zw+Ec1YhtfNiP//14rqJY+S9oCbspNXU+f0CsJRFb14XILvKofd2aDTTYToehdr/fBNums0FjvkdbXwxKSVvj1bsW7jBEuFlWX8cU65C4nsNYHWzuOFd7VDcUhD2eN3x17Nmvi6B3ano82fkpnRQiM31USnpJu+PfnavuGZLfui4SbHyxe7HgKd/TQTE2zeF29zwZvOCnXPDwRJbRgC6s3EaJ2tXD2pu2nI8y7LKXtx/p9Gqbh8enJP37+hL9UNVM8Bjmgs/x2e3uzbBcXhIVbSz7mx45PjfMNaMyJZqI65r0otxyhqLHn2GffZA3zfy9v13B75658n8s2Djvwlm5AvDe/9463922ONcgXl9eXt5d9o5712/Nbw/zb5dnFXv68yxeUGdIZPo/XveEglL03J5c4l0jGl9eX57fwORg9HJD0gbZnr4hMMkOJlCfeol7vJ1aLfMISbwDtrY5j2Fe/rfO3oF//0M8J+As+5GyrsYVMw8tKh5IDdBMX5KdwMjWXQhH2PJaJZlliCJNtvyV7PvNpVTydZ0olQ+1GhWPhJPJEsS2nNl353HQrBNFmKe0DUiePHvub7pETtVbajH55fBzO3X55fEw7rlFcfftWMdzLbnHGkfS7K2oKyMMFz3+B0vDTk8R+HZLYr4+PEA9On5BY1Teccm1s5p1jVHT2yMP7hyXqt5XPhc5hqE2qOw3hDEjtkuirkXozPv7QzoYKrIq/tLMyKcP9ltAfmmAdeJ/WR0jkq+rqpCpBQUoTuydbVBNsFSbyUh3pymc46h7ehHJt19yt69H7467AmftTXoEbfzn6Clw6yl2gMSTHjOR44nPEZanVIy+IRUg34bzWIiyQSr6N1QeDBLHqtoWrIdt+6TB8MhQuZNFfJ3A1VFVSVgAte39JdtjJ2zzdq5GEIxG8KJBxYlFsWSFrLlLZ7IEbvvkLjP1E3ppOzYBLmAqez7YscTWyk6BaV5/VHB+IWMaCPf3Bu9KwSCt/7YSsCl/DQqtT7ckCKBHCVNEynXP6mKZYbGzvgGw2L732bXPGYkAnT+kQi9IuqlNgw9yvWlPP2c1VpT4/VxiPMzxqF0hFYMvtBJTLcHvy/nJ1kaajjuOrfnuU4y/jFDPDuP8LAAD//z2/mF4=" + return "eJzsXF9v27iyf++nGPSpXaQ+e7Z7zkMfLpAmwT0B0jats+ijSpNjmScUqfBPHBf74S/4R7Jsy7FlS85eYPuw20o25/ebGQ6Hw6HfwT0uPgCZm1cAlluBH+A1mZvXrwAYGqp5abmSH+B/XgEA/CBz8wMKxZxAoEoIpNbA+fcxFEpyqzSXORRoNacGploV4d2FUI7NiaWz0SsAjQKJwQ+Qk1cAU46CmQ9h9HcgSYEVGv/HLkr/Qa1cmZ60gFodpDmQJbkZ/VI/rsZTk/8itY3H8UEW397jYq40a3+dFaQsuczTZ1//8rrxuVZs8c8dyf3A8EiEQygJ10k/ZG5Ao1FOUzSjDQa0Vt0Gi6ZWdkj/sRzmB1AlLeHSgJ1hbSs7IxbmqBEM1aREtma97/67MJ9xOlsO0GJzg9LCZAGMT6eo/T88D1OSFXYNJ5igJY3n65Zs6qIeaeVtpY5Ns+1QSjDLDJfDgimR8ilHBvMZSnhwqBcNCwAp+agVWWI/+mXF2WC7w8G60zHlJmKd2L5+twfRT007a7ROy8rGFysEI+t2mowXKA1X0hzMs91KPRJtsIlWaaLemF44MUfNK5yYE00o/8Wrj+ND5lBNlv52HFn62ynJXvzWSjbfN1zQ0o2sskSMyg2fjOQNJQJZNhWKrH9gj7hRoqYoLckR1BSIEIoSi8wDB6qK0lkEJ7lN6iEagTrtY6JYAJfgDIKSQY9cGkskxfZp54lQjYzbzBmSt4c/oWR+QOxzxQS1x39x+wdEIcbHwWiHJjaYKh0+5SwX/Cfxw+7EOyHCf3cQxEhCDGsCj4qWS8wzYoBQqh0yMNw/4RbmxIAgTtIZMlAajCXaIttOxjhdCmeyE5BKolYZzcgjwgRRLi1DJDgpeMG9x9V0w7Llv3Zx+8dFGOFjxJryDm7gJ2q1L1OT0RnROa6H656oBi6thP1ckcr6PIkBU3PpKW/a+wyIZCms2JkzwCV12uuGMMY9CiIgUminLNHOlb4fcTkqCb1HawZhmsYGjRT5o3c66eNFJR64tKinPkVan3TPw1bOngR3CMvK2b5wczmaLCzuDXqqdEHsB2j7UidCYYDezVDRyUrUmUHaky0+J1g1XE0sJgolajBIlWRnQImgToSlZ87tDBh/5Myvquv4PKv3v/7qQ95kAf/+9Q1DS7hA1liK3+72uJc0Xd+eWPMZyHY13kNtVwM8wniMm3uuRhoJO5nxPqaZRlIq6I1VL1PGKo3wqIQr0AB5JFyQiUCwag/TbbDp2XTX7SgDla5W3MB6vBXnmls8sRm9TIvS26d3Ozb4nMaQQeChlmyg7WlCqrKvtftCFaVADz34qipRhyTdHD4D/TaFNMoiJWqumA/Hlhd7zE9V9m3Ub2vUqHLSdp+MquxtKg5hwOijDZoHzrwj7FeT69mA39epdbdgDe0IExpLrDMjOkN6n03Dl3rz0FJpa/xezM5Qr9rD70dLYgwymCg7W30ZMUHAFHY2/q1ZGIvF6jseqwaCGAsFl85uMWULySyOd2KuQxCp5LwAlXaL7UumXhap0v4/TrZXp3xOm6M+uqijNJpQvNm9JtdveUFyHPH2OXFwpf36MoQeD8OPDy4oVKViTBd8y/k98jbo8UTgWjJOSch6kie0BBXgBlD6iLulalQDLTV/JBZHTJrMv+pXoWl0uPw8DoIr9W5si/ZEyct2T1x/3AHa9e3j70AY02gMEGMU5cvIfhBWNxGcDqXQMPiGPvf0ygStRy1Wiks4rnxw4RSub+s3b7yC38JEOcmq5b+rSsMUGlHF2rV5cCAK467r8AyIAQL//Pe7CbfgpOG5DDXMIGQvpP3bvRUpvClRhsTjT9BOyvg3M3PWcpm/C3XJP8GiLrgMPv2nz8vC6VX1V2Rb8o+akZ35hDRmWT5UD7UUJDkh0aqWhZbzMDF5tS680xGRmJzyiOjm4zHnYZodd/in2SkP/75dHnT4B32diFVVhHTsdUBsXTkv63qS9PfJ16lPvhixZEIMZlRJiTTs1gahUwmChqB0QLoF2aTO/kdEy/6WgvOC/FQSvqV+HPhMCnxz/u3z2+ACSOjMB43doKggpl1XB8G6aMaYZl5SHTP7zWKBhdILoKQklNsFBAzVBy8/7qrGNdBzhtLyKd9YcPqgQLxZ9TvjylJwZEvjL6WO4G7GTeOBT7c9Cyf5g8PQRxX8vf6EH7YTxbhx64/eOJVYIs50zN/MKripmW4vu2QPDh1mDEs7G2SmKWe9AkJOc/3FwBufE/wjVUTxwaGx5i3MCfcJTig6UeqTTE/KA2yHXlUWHkRmUD+izkiO0mb/VZNhAkYUCOOvNzAOAuHcCwQvEJgLS+heW/GpRvS7uCxOnpMebZEiFL7UtFG+00QyVVRaT6C2Is+MVdrv018adsIBoYtuS4cceeKFKzK/5c+sJtKQEOgzzvr0kSQGGhLg+rLqojCxicJjGMF5CEChtHqrjM01jr/etINXgqGxmcZScBqW/8wIZTNB8lEx6RG+IHnundfwn3WMT1LrdyHPVMaGehPqIsT47+c3Ib7Uh8Wd+PkokHG1s9p7YKMUecTgHo0Vn5v7eEpx/Y8vzWLwsgy8DWlQRtB8h5p05fPMRUHHuL3lBQKJBxHfkm0aa4+3k/ezGa8KuDGVaC5PTdt8Woy/3pzBJ6I5ufwYO1qW9loRsyXxMHNSxvT4hQKBBxDnfqzopaa2FcZhRU8NmdyE5p5l/PDJ1TKYt7NsxgyhcpPlKLHVmsdMwOCYDSp+J9AIJV5wp5kVltbTT624onecWw8ONd/ffQ5Cl2QAPiF1Np5IPQuKIWFC0fthYdVSqiJ+nZXuwheP4MKy9lKzLy2+lb8ulNNw7rTSK3HpzFMLwp5JKISiRLxwWlHxWQ0PFotSaaIXYP0zE+Kkn467eAmVcxnOjJwe2LlTWhokArEest097exMK5fPSmdHVBUFby/M9IUxiugSFhr4GArcctbS30z0Iuo40QUcE4Miu7y8qfdInXAVw+Li0qC25gxcyYjF1BQb9dgJaBzoBFgPsW460ugTXR1xquOSpbh4OF83Xsb+GZ/P+Vyg4NbGc0sqOEprYnMyna20XfhVJEXhkOL5WEyFMxb1MmQdoIEsoepRE1xSVfitxZtvcey3S5VoMp1y2pLSeRJUuFBKCNqizlhVoF6undWXveaqytrluH4cFiwf2xuFb//RxjZrb6VUhulRK8rZXAWt3KXB//+oxW+tBpjIa5sgS+5RBhcOSedKurkTokGxeVWs32gTRRwSbWIoHRRcFHEIuJDcD4ptst71Gey7C6IgFiVddMxi+tyaJwRh9mwkOiHqFlwIHklsyxcTiw7pxFAUQkGH4ZRLHnefRObOW+rN5eXN2zoZ6Ups/3xkKGLPpiwd6XTLWoZlVM3mjhS6ROseCPQVzCv43SL5UBZYDfYdLdAt3g9FYXVJ6Eih06rwF/KicO6Vqpc8lPZeqO7QKP0pSl3JYzllwiXRi1BqqNK9gvgsfrOKm+6IPFusbdBdP07o9yihpZLZEAheIEy5wG71zAb89YLs4PCPKsQ2vmxG/v+97qBa6izpCLgpN1U9fUKvJBBZ7Q+XR+DV/nFnNthkMxGK3vd6R3eTzgqN9RppfWU3IWmFX59WvM8Y4WJRVRlfrUPu0oG1PthaO1Z4VxcUh2zOGr8/tjdr4ug92p5bGz+nXhEC4/eVhOekG/7zpcq+Idmt6yLhfguL11eewx09NFPTLP6ESJ8F3tAp12yeiBJasIXVmwhRu1rovWn7OR/zPktp+7F+n4ZpeHx68refP+Mv1Z4ptkEO6Cz/ubu7XZaLC8LC3Swf82PFp8Z5Bhpzopmo2rwX5ZYWihp7jn3WTdYw/+/V3Rpu79yV73PZxmEH3tINiPf2j97x9n7MsQb58urm6u6qb9Szfmt+a5j/c3V+uZc/7/IFZYZ0hi/jdW84CGXvxcklziWS8dXN1cUdfAlGDw2SPtD27BWRSWYokfLER9Tr9cRqkU9Y4g2gvdVxDPvq987+EvTrH187AX/Bh5xtNbaQaXhZqSk5QDdxQX4OJ1NzKRRhL2OZaJYlhjDZ9luy5zOfVsXuPFMqGfZuVDgWOpEnim3p2nTlS9OtEESbpbQPSJ08euxn3SMnaq20Gf3+9DScu/3+9JROXKO4+vatYriX3eKMI+m3sNQUkIcLnuEC+T+fJfavIYn96+kJYuP0CYlVdcMp18Zm3jlGRWePPLx+WKJ+V/lcqByGvUl1pyH0gNQuiX43Uh/Gxx8/21CBVfHXz1YmZbjfEupDE6wD7/P6CIl8tbs6qUpQkNLE6skW1QRbhYm8VEe68hla3cObsF3bNXfr/ejDcVfgzMMpr8CNvx59BS61chdoDMkxIzmeuI+4LLV64gWxCOkmnNdahAVSyXdx98EgQayqbeFqyLZfnw2fDBsXsuivErgaqiopK4CWtb8kO5zkbXb3aiShJYIXBTJOLIotK2TNRSqbPXLDN38Vt5/IW9OpGXAJU8Hz2ZYlrkZ2ElTr6rOa4yMRy1iwpz94VxoWaeWvnZBV4WtYaHWqPVkAJUKYKlqmPqdPaYrFwvYOyGbz0mvfNmcsBnTynA6xKO2i6gIb5n7VmnrOb68r9fm5wnic4VG7QCoCW24noFyG25PXl6uLNB11HF/1W6Mcfx2nmBnG/b8AAAD//zHbSRk=" } diff --git a/x-pack/metricbeat/module/aws/mtest/integration.go b/x-pack/metricbeat/module/aws/mtest/integration.go index aabc2f5d0a0b..34f143830c06 100644 --- a/x-pack/metricbeat/module/aws/mtest/integration.go +++ b/x-pack/metricbeat/module/aws/mtest/integration.go @@ -38,6 +38,7 @@ func GetConfigForTest(metricSetName string, period string) (map[string]interface "access_key_id": accessKeyID, "secret_access_key": secretAccessKey, "default_region": defaultRegion, + "regions": []string{"us-east-1"}, } if okSessionToken && sessionToken != "" { diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 5a3241631b57..68ef761800ec 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -7,6 +7,7 @@ - ec2 regions: - us-east-1 + calculate_rate: false - module: aws period: 300s metricsets: From f7d9d20495092d71d89975535c995ae70bc45e84 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 16 Aug 2019 14:48:45 -0600 Subject: [PATCH 2/4] Remove calculate_rate flag and add a test --- metricbeat/docs/fields.asciidoc | 32 ++++++-- .../metricbeat/module/aws/ec2/_meta/data.json | 18 +++-- .../module/aws/ec2/_meta/docs.asciidoc | 15 ---- .../module/aws/ec2/_meta/fields.yml | 20 +++-- x-pack/metricbeat/module/aws/ec2/ec2.go | 60 +++------------ x-pack/metricbeat/module/aws/ec2/ec2_test.go | 74 +++++++++++++++++++ x-pack/metricbeat/module/aws/fields.go | 2 +- 7 files changed, 137 insertions(+), 84 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index fcd3fea2fa9b..a732e08253ed 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -950,6 +950,16 @@ type: long The number of packets received on all network interfaces by the instance. +type: long + +-- + +*`aws.ec2.network.in.packets_per_sec`*:: ++ +-- +The number of packets per second sent out on all network interfaces by the instance. + + type: long -- @@ -960,6 +970,16 @@ type: long The number of packets sent out on all network interfaces by the instance. +type: long + +-- + +*`aws.ec2.network.out.packets_per_sec`*:: ++ +-- +The number of packets per second sent out on all network interfaces by the instance. + + type: long -- @@ -979,7 +999,7 @@ format: bytes *`aws.ec2.network.in.bytes_per_sec`*:: + -- -Network received rate bytes per second, calculated with dividing network.in.bytes by 300 or by 60(detailed monitoring). +The number of bytes per second received on all network interfaces by the instance. type: long @@ -1001,7 +1021,7 @@ format: bytes *`aws.ec2.network.out.bytes_per_sec`*:: + -- -Network sent out rate bytes per second, calculated with dividing network.out.bytes by 300 or by 60(detailed monitoring). +The number of bytes per second sent out on all network interfaces by the instance. type: long @@ -1023,7 +1043,7 @@ format: bytes *`aws.ec2.diskio.read.bytes_per_sec`*:: + -- -Instance store volumes read rate bytes per second, calculated with dividing diskio.read.bytes by 300 or by 60(detailed monitoring). +Bytes read per second from all instance store volumes available to the instance. type: long @@ -1045,7 +1065,7 @@ format: bytes *`aws.ec2.diskio.write.bytes_per_sec`*:: + -- -Instance store volumes write rate bytes per second, calculated with dividing diskio.write.bytes by 300 or by 60(detailed monitoring). +Bytes written per second to all instance store volumes available to the instance. type: long @@ -1065,7 +1085,7 @@ type: long *`aws.ec2.diskio.read.ops_per_sec`*:: + -- -Read operations count per second, calculated with dividing diskio.read.ops by 300 or by 60(detailed monitoring). +Completed read operations per second from all instance store volumes available to the instance in a specified period of time. type: long @@ -1085,7 +1105,7 @@ type: long *`aws.ec2.diskio.write.ops_per_sec`*:: + -- -Write operations count per second, calculated with dividing diskio.write.ops by 300 or by 60(detailed monitoring). +Completed write operations per second to all instance store volumes available to the instance in a specified period of time. type: long diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/data.json b/x-pack/metricbeat/module/aws/ec2/_meta/data.json index 455401a90709..be9324a9098a 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/data.json +++ b/x-pack/metricbeat/module/aws/ec2/_meta/data.json @@ -4,11 +4,11 @@ "ec2": { "cpu": { "credit_balance": 576, - "credit_usage": 0.194305, + "credit_usage": 0.186525, "surplus_credit_balance": 0, "surplus_credits_charged": 0, "total": { - "pct": 2.0499999999883576 + "pct": 2.0000740946562456 } }, "diskio": { @@ -51,14 +51,16 @@ }, "network": { "in": { - "bytes": 1297480, - "bytes_per_sec": 4324.933333333333, - "packets": 2734.8 + "bytes": 1368258.2, + "bytes_per_sec": 4560.8606666666665, + "packets": 2904.6, + "packets_per_sec": 9.682 }, "out": { - "bytes": 395706.4, - "bytes_per_sec": 1319.0213333333334, - "packets": 2672.8 + "bytes": 425826.8, + "bytes_per_sec": 1419.4226666666666, + "packets": 2910, + "packets_per_sec": 9.7 } }, "status": { diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc index 3e892db4f8af..5428dfb80446 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc @@ -47,20 +47,6 @@ The aws ec2 metricset comes with a predefined dashboard. For example: image::./images/metricbeat-aws-ec2-overview.png[] -[float] -=== Metricset-specific configuration notes -* *calculate_rate*: If calculate_rate is true, 6 rate metrics will be calculated: -|=== -|Rate Metric Name | Equation | Equation with Detailed Monitoring -|network.in.bytes_per_sec | network.in.bytes / 300 | network.in.bytes / 60 -|network.out.bytes_per_sec | network.out.bytes / 300 | network.out.bytes / 60 -|diskio.read.bytes_per_sec | diskio.read.bytes / 300 | diskio.read.bytes / 60 -|diskio.write.bytes_per_sec | diskio.write.bytes / 300 | diskio.write.bytes / 60 -|diskio.read.ops_per_sec | diskio.read.ops / 300 | diskio.read.ops / 60 -|diskio.write.ops_per_sec | diskio.write.ops / 300 | diskio.write.ops / 60 -|=== -Default calculate_rate is false. - [float] === Configuration example [source,yaml] @@ -72,5 +58,4 @@ Default calculate_rate is false. access_key_id: '' secret_access_key: '' session_token: '' - calculate_rate: false ---- diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml b/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml index c856c980b17e..75fe4e9bf392 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/ec2/_meta/fields.yml @@ -28,10 +28,18 @@ type: long description: > The number of packets received on all network interfaces by the instance. + - name: network.in.packets_per_sec + type: long + description: > + The number of packets per second sent out on all network interfaces by the instance. - name: network.out.packets type: long description: > The number of packets sent out on all network interfaces by the instance. + - name: network.out.packets_per_sec + type: long + description: > + The number of packets per second sent out on all network interfaces by the instance. - name: network.in.bytes type: long format: bytes @@ -40,7 +48,7 @@ - name: network.in.bytes_per_sec type: long description: > - Network received rate bytes per second, calculated with dividing network.in.bytes by 300 or by 60(detailed monitoring). + The number of bytes per second received on all network interfaces by the instance. - name: network.out.bytes type: long format: bytes @@ -49,7 +57,7 @@ - name: network.out.bytes_per_sec type: long description: > - Network sent out rate bytes per second, calculated with dividing network.out.bytes by 300 or by 60(detailed monitoring). + The number of bytes per second sent out on all network interfaces by the instance. - name: diskio.read.bytes type: long format: bytes @@ -58,7 +66,7 @@ - name: diskio.read.bytes_per_sec type: long description: > - Instance store volumes read rate bytes per second, calculated with dividing diskio.read.bytes by 300 or by 60(detailed monitoring). + Bytes read per second from all instance store volumes available to the instance. - name: diskio.write.bytes type: long format: bytes @@ -67,7 +75,7 @@ - name: diskio.write.bytes_per_sec type: long description: > - Instance store volumes write rate bytes per second, calculated with dividing diskio.write.bytes by 300 or by 60(detailed monitoring). + Bytes written per second to all instance store volumes available to the instance. - name: diskio.read.ops type: long description: > @@ -75,7 +83,7 @@ - name: diskio.read.ops_per_sec type: long description: > - Read operations count per second, calculated with dividing diskio.read.ops by 300 or by 60(detailed monitoring). + Completed read operations per second from all instance store volumes available to the instance in a specified period of time. - name: diskio.write.ops type: long description: > @@ -83,7 +91,7 @@ - name: diskio.write.ops_per_sec type: long description: > - Write operations count per second, calculated with dividing diskio.write.ops by 300 or by 60(detailed monitoring). + Completed write operations per second to all instance store volumes available to the instance in a specified period of time. - name: status.check_failed type: long description: > diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 9a21aabe3bc4..32371ceb163b 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -44,7 +44,6 @@ func init() { // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet - CalculateRate bool `config:"calculate_rate"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -65,18 +64,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { base.Logger().Info(err) } - config := struct { - CalculateRate bool `config:"calculate_rate"` - }{} - - err = base.Module().UnpackConfig(&config) - if err != nil { - return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") - } - return &MetricSet{ - MetricSet: metricSet, - CalculateRate: config.CalculateRate, + MetricSet: metricSet, }, nil } @@ -245,9 +234,8 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met return events, errors.Wrap(err, "EventMapping failed") } - if m.CalculateRate { - calculateRate(resultMetricsetFields, monitoringStates[instanceID]) - } + // add rate metrics + calculateRate(resultMetricsetFields, monitoringStates[instanceID]) events[instanceID].MetricSetFields.Update(resultMetricsetFields) if len(events[instanceID].MetricSetFields) < 5 { @@ -267,40 +255,16 @@ func calculateRate(resultMetricsetFields common.MapStr, monitoringState string) period = 60.0 } - networkIn, err := resultMetricsetFields.GetValue("network.in.bytes") - if err == nil && networkIn != nil { - networkInRate := networkIn.(float64) / period - resultMetricsetFields.Put("network.in.bytes_per_sec", networkInRate) - } - - networkOut, err := resultMetricsetFields.GetValue("network.out.bytes") - if err == nil && networkOut != nil { - networkOutRate := networkOut.(float64) / period - resultMetricsetFields.Put("network.out.bytes_per_sec", networkOutRate) - } + metricList := []string{"network.in.bytes", "network.out.bytes", + "network.in.packets", "network.out.packets", "diskio.read.bytes", + "diskio.write.bytes", "diskio.read.count", "diskio.write.count"} - diskReadBytes, err := resultMetricsetFields.GetValue("diskio.read.bytes") - if err == nil && networkOut != nil { - diskReadBytesRate := diskReadBytes.(float64) / period - resultMetricsetFields.Put("diskio.read.bytes_per_sec", diskReadBytesRate) - } - - diskWriteBytes, err := resultMetricsetFields.GetValue("diskio.write.bytes") - if err == nil && networkOut != nil { - diskWriteBytesRate := diskWriteBytes.(float64) / period - resultMetricsetFields.Put("diskio.write.bytes_per_sec", diskWriteBytesRate) - } - - diskReadOps, err := resultMetricsetFields.GetValue("diskio.read.count") - if err == nil && networkOut != nil { - diskReadOpsRate := diskReadOps.(float64) / period - resultMetricsetFields.Put("diskio.read.count_per_sec", diskReadOpsRate) - } - - diskWriteOps, err := resultMetricsetFields.GetValue("diskio.write.count") - if err == nil && networkOut != nil { - diskWriteOpsRate := diskWriteOps.(float64) / period - resultMetricsetFields.Put("diskio.write.count_per_sec", diskWriteOpsRate) + for _, metricName := range metricList { + metricValue, err := resultMetricsetFields.GetValue(metricName) + if err == nil && metricValue != nil { + rateValue := metricValue.(float64) / period + resultMetricsetFields.Put(metricName+"_per_sec", rateValue) + } } } diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 89049907cb19..bc8eed410a72 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -221,3 +221,77 @@ func TestConstructMetricQueries(t *testing.T) { assert.Equal(t, metricName1, *metricDataQuery[0].MetricStat.Metric.MetricName) assert.Equal(t, namespace, *metricDataQuery[0].MetricStat.Metric.Namespace) } + +func TestCalculateRate(t *testing.T) { + resultMetricsetFields := common.MapStr{ + "network.in.bytes": 1367316.0, + "network.out.bytes": 427380.0, + "network.in.packets": 2895.0, + "network.out.packets": 2700.0, + "diskio.read.bytes": 300.0, + "diskio.write.bytes": 600.0, + "diskio.read.count": 30.0, + "diskio.write.count": 60.0, + } + + cases := []struct { + rateMetricName string + rateMetricValue float64 + rateMetricValueDetailed float64 + }{ + { + "network.in.bytes_per_sec", + 4557.72, + 22788.6, + }, + { + "network.out.bytes_per_sec", + 1424.6, + 7123.0, + }, + { + "network.in.packets_per_sec", + 9.65, + 48.25, + }, + { + "network.out.packets_per_sec", + 9.0, + 45.0, + }, + { + "diskio.read.bytes_per_sec", + 1.0, + 5.0, + }, + { + "diskio.write.bytes_per_sec", + 2.0, + 10.0, + }, + { + "diskio.read.count_per_sec", + 0.1, + 0.5, + }, + { + "diskio.write.count_per_sec", + 0.2, + 1.0, + }, + } + + calculateRate(resultMetricsetFields, "disabled") + for _, c := range cases { + output, err := resultMetricsetFields.GetValue(c.rateMetricName) + assert.NoError(t, err) + assert.Equal(t, c.rateMetricValue, output) + } + + calculateRate(resultMetricsetFields, "enabled") + for _, c := range cases { + output, err := resultMetricsetFields.GetValue(c.rateMetricName) + assert.NoError(t, err) + assert.Equal(t, c.rateMetricValueDetailed, output) + } +} diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index b0c714e253f6..8a8ecaadb5c4 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsXF9v27iyf++nGPSpXaQ+e7Z7zkMfLpAmwT0B0jats+ijSpNjmScUqfBPHBf74S/4R7Jsy7FlS85eYPuw20o25/ebGQ6Hw6HfwT0uPgCZm1cAlluBH+A1mZvXrwAYGqp5abmSH+B/XgEA/CBz8wMKxZxAoEoIpNbA+fcxFEpyqzSXORRoNacGploV4d2FUI7NiaWz0SsAjQKJwQ+Qk1cAU46CmQ9h9HcgSYEVGv/HLkr/Qa1cmZ60gFodpDmQJbkZ/VI/rsZTk/8itY3H8UEW397jYq40a3+dFaQsuczTZ1//8rrxuVZs8c8dyf3A8EiEQygJ10k/ZG5Ao1FOUzSjDQa0Vt0Gi6ZWdkj/sRzmB1AlLeHSgJ1hbSs7IxbmqBEM1aREtma97/67MJ9xOlsO0GJzg9LCZAGMT6eo/T88D1OSFXYNJ5igJY3n65Zs6qIeaeVtpY5Ns+1QSjDLDJfDgimR8ilHBvMZSnhwqBcNCwAp+agVWWI/+mXF2WC7w8G60zHlJmKd2L5+twfRT007a7ROy8rGFysEI+t2mowXKA1X0hzMs91KPRJtsIlWaaLemF44MUfNK5yYE00o/8Wrj+ND5lBNlv52HFn62ynJXvzWSjbfN1zQ0o2sskSMyg2fjOQNJQJZNhWKrH9gj7hRoqYoLckR1BSIEIoSi8wDB6qK0lkEJ7lN6iEagTrtY6JYAJfgDIKSQY9cGkskxfZp54lQjYzbzBmSt4c/oWR+QOxzxQS1x39x+wdEIcbHwWiHJjaYKh0+5SwX/Cfxw+7EOyHCf3cQxEhCDGsCj4qWS8wzYoBQqh0yMNw/4RbmxIAgTtIZMlAajCXaIttOxjhdCmeyE5BKolYZzcgjwgRRLi1DJDgpeMG9x9V0w7Llv3Zx+8dFGOFjxJryDm7gJ2q1L1OT0RnROa6H656oBi6thP1ckcr6PIkBU3PpKW/a+wyIZCms2JkzwCV12uuGMMY9CiIgUminLNHOlb4fcTkqCb1HawZhmsYGjRT5o3c66eNFJR64tKinPkVan3TPw1bOngR3CMvK2b5wczmaLCzuDXqqdEHsB2j7UidCYYDezVDRyUrUmUHaky0+J1g1XE0sJgolajBIlWRnQImgToSlZ87tDBh/5Myvquv4PKv3v/7qQ95kAf/+9Q1DS7hA1liK3+72uJc0Xd+eWPMZyHY13kNtVwM8wniMm3uuRhoJO5nxPqaZRlIq6I1VL1PGKo3wqIQr0AB5JFyQiUCwag/TbbDp2XTX7SgDla5W3MB6vBXnmls8sRm9TIvS26d3Ozb4nMaQQeChlmyg7WlCqrKvtftCFaVADz34qipRhyTdHD4D/TaFNMoiJWqumA/Hlhd7zE9V9m3Ub2vUqHLSdp+MquxtKg5hwOijDZoHzrwj7FeT69mA39epdbdgDe0IExpLrDMjOkN6n03Dl3rz0FJpa/xezM5Qr9rD70dLYgwymCg7W30ZMUHAFHY2/q1ZGIvF6jseqwaCGAsFl85uMWULySyOd2KuQxCp5LwAlXaL7UumXhap0v4/TrZXp3xOm6M+uqijNJpQvNm9JtdveUFyHPH2OXFwpf36MoQeD8OPDy4oVKViTBd8y/k98jbo8UTgWjJOSch6kie0BBXgBlD6iLulalQDLTV/JBZHTJrMv+pXoWl0uPw8DoIr9W5si/ZEyct2T1x/3AHa9e3j70AY02gMEGMU5cvIfhBWNxGcDqXQMPiGPvf0ygStRy1Wiks4rnxw4RSub+s3b7yC38JEOcmq5b+rSsMUGlHF2rV5cCAK467r8AyIAQL//Pe7CbfgpOG5DDXMIGQvpP3bvRUpvClRhsTjT9BOyvg3M3PWcpm/C3XJP8GiLrgMPv2nz8vC6VX1V2Rb8o+akZ35hDRmWT5UD7UUJDkh0aqWhZbzMDF5tS680xGRmJzyiOjm4zHnYZodd/in2SkP/75dHnT4B32diFVVhHTsdUBsXTkv63qS9PfJ16lPvhixZEIMZlRJiTTs1gahUwmChqB0QLoF2aTO/kdEy/6WgvOC/FQSvqV+HPhMCnxz/u3z2+ACSOjMB43doKggpl1XB8G6aMaYZl5SHTP7zWKBhdILoKQklNsFBAzVBy8/7qrGNdBzhtLyKd9YcPqgQLxZ9TvjylJwZEvjL6WO4G7GTeOBT7c9Cyf5g8PQRxX8vf6EH7YTxbhx64/eOJVYIs50zN/MKripmW4vu2QPDh1mDEs7G2SmKWe9AkJOc/3FwBufE/wjVUTxwaGx5i3MCfcJTig6UeqTTE/KA2yHXlUWHkRmUD+izkiO0mb/VZNhAkYUCOOvNzAOAuHcCwQvEJgLS+heW/GpRvS7uCxOnpMebZEiFL7UtFG+00QyVVRaT6C2Is+MVdrv018adsIBoYtuS4cceeKFKzK/5c+sJtKQEOgzzvr0kSQGGhLg+rLqojCxicJjGMF5CEChtHqrjM01jr/etINXgqGxmcZScBqW/8wIZTNB8lEx6RG+IHnundfwn3WMT1LrdyHPVMaGehPqIsT47+c3Ib7Uh8Wd+PkokHG1s9p7YKMUecTgHo0Vn5v7eEpx/Y8vzWLwsgy8DWlQRtB8h5p05fPMRUHHuL3lBQKJBxHfkm0aa4+3k/ezGa8KuDGVaC5PTdt8Woy/3pzBJ6I5ufwYO1qW9loRsyXxMHNSxvT4hQKBBxDnfqzopaa2FcZhRU8NmdyE5p5l/PDJ1TKYt7NsxgyhcpPlKLHVmsdMwOCYDSp+J9AIJV5wp5kVltbTT624onecWw8ONd/ffQ5Cl2QAPiF1Np5IPQuKIWFC0fthYdVSqiJ+nZXuwheP4MKy9lKzLy2+lb8ulNNw7rTSK3HpzFMLwp5JKISiRLxwWlHxWQ0PFotSaaIXYP0zE+Kkn467eAmVcxnOjJwe2LlTWhokArEest097exMK5fPSmdHVBUFby/M9IUxiugSFhr4GArcctbS30z0Iuo40QUcE4Miu7y8qfdInXAVw+Li0qC25gxcyYjF1BQb9dgJaBzoBFgPsW460ugTXR1xquOSpbh4OF83Xsb+GZ/P+Vyg4NbGc0sqOEprYnMyna20XfhVJEXhkOL5WEyFMxb1MmQdoIEsoepRE1xSVfitxZtvcey3S5VoMp1y2pLSeRJUuFBKCNqizlhVoF6undWXveaqytrluH4cFiwf2xuFb//RxjZrb6VUhulRK8rZXAWt3KXB//+oxW+tBpjIa5sgS+5RBhcOSedKurkTokGxeVWs32gTRRwSbWIoHRRcFHEIuJDcD4ptst71Gey7C6IgFiVddMxi+tyaJwRh9mwkOiHqFlwIHklsyxcTiw7pxFAUQkGH4ZRLHnefRObOW+rN5eXN2zoZ6Ups/3xkKGLPpiwd6XTLWoZlVM3mjhS6ROseCPQVzCv43SL5UBZYDfYdLdAt3g9FYXVJ6Eih06rwF/KicO6Vqpc8lPZeqO7QKP0pSl3JYzllwiXRi1BqqNK9gvgsfrOKm+6IPFusbdBdP07o9yihpZLZEAheIEy5wG71zAb89YLs4PCPKsQ2vmxG/v+97qBa6izpCLgpN1U9fUKvJBBZ7Q+XR+DV/nFnNthkMxGK3vd6R3eTzgqN9RppfWU3IWmFX59WvM8Y4WJRVRlfrUPu0oG1PthaO1Z4VxcUh2zOGr8/tjdr4ug92p5bGz+nXhEC4/eVhOekG/7zpcq+Idmt6yLhfguL11eewx09NFPTLP6ESJ8F3tAp12yeiBJasIXVmwhRu1rovWn7OR/zPktp+7F+n4ZpeHx68refP+Mv1Z4ptkEO6Cz/ubu7XZaLC8LC3Swf82PFp8Z5Bhpzopmo2rwX5ZYWihp7jn3WTdYw/+/V3Rpu79yV73PZxmEH3tINiPf2j97x9n7MsQb58urm6u6qb9Szfmt+a5j/c3V+uZc/7/IFZYZ0hi/jdW84CGXvxcklziWS8dXN1cUdfAlGDw2SPtD27BWRSWYokfLER9Tr9cRqkU9Y4g2gvdVxDPvq987+EvTrH187AX/Bh5xtNbaQaXhZqSk5QDdxQX4OJ1NzKRRhL2OZaJYlhjDZ9luy5zOfVsXuPFMqGfZuVDgWOpEnim3p2nTlS9OtEESbpbQPSJ08euxn3SMnaq20Gf3+9DScu/3+9JROXKO4+vatYriX3eKMI+m3sNQUkIcLnuEC+T+fJfavIYn96+kJYuP0CYlVdcMp18Zm3jlGRWePPLx+WKJ+V/lcqByGvUl1pyH0gNQuiX43Uh/Gxx8/21CBVfHXz1YmZbjfEupDE6wD7/P6CIl8tbs6qUpQkNLE6skW1QRbhYm8VEe68hla3cObsF3bNXfr/ejDcVfgzMMpr8CNvx59BS61chdoDMkxIzmeuI+4LLV64gWxCOkmnNdahAVSyXdx98EgQayqbeFqyLZfnw2fDBsXsuivErgaqiopK4CWtb8kO5zkbXb3aiShJYIXBTJOLIotK2TNRSqbPXLDN38Vt5/IW9OpGXAJU8Hz2ZYlrkZ2ElTr6rOa4yMRy1iwpz94VxoWaeWvnZBV4WtYaHWqPVkAJUKYKlqmPqdPaYrFwvYOyGbz0mvfNmcsBnTynA6xKO2i6gIb5n7VmnrOb68r9fm5wnic4VG7QCoCW24noFyG25PXl6uLNB11HF/1W6Mcfx2nmBnG/b8AAAD//zHbSRk=" + return "eJzsXF1z27bSvs+v2MlV0knUt037XuTizDi259QzTuJE7vSSgYAVhRoEaHxYVqY//gw+SFESZYkSKbczzUWbkBL2eXYXi8Viobdwh4v3QObmBYDlVuB7eEnm5uULAIaGal5aruR7+M8LAIBvZG6+QaGYEwhUCYHUGjj7YwyFktwqzWUOBVrNqYGpVkV4dy6UY3Ni6Wz0AkCjQGLwPeTkBcCUo2DmfRj9LUhSYIXG/7GL0n9QK1emJy2gVgdpDmRJbkY/1I+r8dTkT6S28Tg+yOLbO1zMlWbtr7OClCWXefrsyx9eNj7Xii3+uSW5HxgeiHAIJeE66YfMDWg0ymmKZrTBgNaq22DR1MoO6d+Ww3wDqqQlXBqwM6xtZWfEwhw1gqGalMjWrPeH/y7MZ5zOlgO02NygtDBZAOPTKWr/D8/DlGSFXcMJJmhJ4/m6JZu6qEdaeVupY9NsO5QSzDLD5bBgSqR8ypHBfIYS7h3qRcMCQEo+akWW2I9+WHE22O5wsO50TLmJWCe2r9/tQfRj084ardOysvH5CsHIup0m4wVKw5U0B/Nst1KPRBtsolWaqDemF07MUfMKJ+ZEE8p/8fLD+JA5VJOlPx9Hlv58SrLnP7eSzfcNF7R0I6ssEaNywycjeUOJQJZNhSLrH9gjbpSoKUpLcgQ1BSKEosQi88CBqqJ0FsFJbpN6iEagTvuYKBbAJTiDoGTQI5fGEkmxfdp5IlQj4zZzhuTt4U8omR8Q+1wxQe3xn9/8DlGI8XEw2qGJDaZKh085ywX/TvywO/FOiPDfHQQxkhDDmsCjouUS84wYIJRqhwwM90+4hTkxIIiTdIYMlAZjibbItpMxTpfCmewEpJKoVUYz8oAwQZRLyxAJTgpecO9xNd2wbPmvnd/8fh5G+BCxpryDG/iOWu3L1GR0RnSO6+G6J6qBSythP1eksj5PYsDUXHrKm/Z+A0SyFFbszBngkjrtdUMY4x4FERAptFOWaOdK3424HJWE3qE1gzBNY4NGivzBO5308aISD1xa1FOfIq1Pun1hZyXqzCAdFH6JGgxSJVkM1MrZvpgoZ09igQFx/9NNwOVosrC4t/6nShfEvoe2L3XiFgYYYm6EgQc1S4TeMErfLLx/PadVhpgvz2CWvmgwbu64Gmkk7GRm+ZCmB0lJtcdfL/jGKo3woIQr0AB5IFyQiUCw6hA2PRulgbxhi2FIzDW3eGKbeJkWpcc5JJ9BrFJhbxhmCBrBt1TZ16J+ropSoE95g1epEnXYh5jDvcrvxEij8lOi5or5KGJ5sR+5ng20nWQfs+gIvtEnh7BmGLnJ9EBf7IPcYNbcIHn83DuEr7HEOjOiM6R32ZRw0dv27iuWSlvjd6F2hnoVqd+Jl8QYZDBRdrb6MmKCgCns6fxbszAWi9V3PNZLBDEWCi6d3Z9kFsc7MdchiFRynoFKu8X2JVMvGVRp/x8n2+tyPi3LUR9dzlIaTShb7V6v6re8IDmOePucOPiM4eoiTEoPw48PLihUpTJUF3zLqunI26DHs5AryTglITlInsDQBo9rlmq5AZQ+Fm2pl9VAS80fiMURkybzr/pVaBodLj6Ng+BKvRuZ/Z4oednuieuPO0C7unn4BQhjGo0BYoyiPNSH5zyFv85Y3URwOpRCw+Ab+tzTKxO0HrVYKS7huPTBhVO4uqnfvPIKfg0T5eICeohKwxQaUcXatXlwIArjruvwDRADBH76/7cTbsFJw3MZqrdByF5I+7d7K1J4VaJkfrr/BdpJGf9mZs5aLvO3oSL7F1jUBZfBp//yGUs4t6v+iuz1DkZ25vPbmG/5UD3UUpDkhHSrWhZaTgLF5MW68E6HY2JyysOx6w/HnARqdtyxp2anPPb8enHQsSf0dRZYFQrSgd8BsXXlpLDrGdq/Z36nPvNjxJIJMZhRJSXSsFsbhE4lCBqC0tHwFmSTOvsfES37WwrOCvJdSfiaOpHgEynw1dnXT6+DCyChMx80doOigph2XR0E67wZY5p5SXXA7jeLBRZKL4CSklBuFxAwVB+8+LCrUtVAzxlK67fS6wtOHxSIN6t+a1xZCr9Zr42/lDqC2xk3jQc+3fYsnOT3DkMHWfD3+hN+2E4U48atP3rjVHyIOFODQzOr4KZmur0Ak907dJgxLO1skJmmnPUKCDnN1WcDr3xO8GMsyWi8d2iseQ1zwn2CE8oxlPok05PyANuhV5WFe5EZ1A+oM5KjtNmfajJMwIgCYfzlGsZBIJx5geAFAnNhCd1rKz7ViH4Xl8XJc9JDJlL4nX9omqkLW5pIpopK6wnUVuSZsUr7ffpzw044IPQPbukNJI+8cEXmt/yZ1UQaEgJ9xlmfPpLEQEMCXF1U/SMmto94DCM4CwEoFFlvlLG5xvGX63bwSjA0NtNYCk7D8p8ZoWwmSD4qJj3CFyTPvfMa/r2O8Ulq/S7kmcrYUG9CXYQY/8fZdYgv9dFrJ34+CmRc7az7HtgiRh4wuEdjxefmLtb1r3783F4M3oY0KCNovkN1uvJ55qKgY9ze8gKBwFeP/muyTWPt8XbyfjbjVQE3phLN5alpm4+L8ZfrN/CRaE4uPsRenqW9VsRsSTzMnJQxPX6mQOABxLkfK3qpnW+FcVjRUysqN6GtaRk/fHK1DObtLJsxQ6jcZDlKbLXmMRMwOGaDit8JNEKJF9xpZoWl9fRTK67oHefWvUPN93efg9AlGYCPSJ1FthMUQ8KEonfDwqqlVEX8OivdhS8eToVl7blmX1p8K39dKKfhzGmlV+LSG08tCHsioRCKEvHMaUXFZzU8WCxKpYlegPXPTIiTfjru4iVUzmU4M3J6YOdOaWmQCMR6yHb3tLMzrVw+K50dUVUUvL0w0xfGKKJLWGjgYyhwy1lLfzPRi6jjRBdwTAyK7OLiut4jdcJVDIuLS4PamjfgSkYspnbgqMdOQONAJ8B6iHXTkUaf6OqIUx2XLMXFw/m6jTG2mfh8zucCBbc2nltSwVFaE9uy6WylIcGvIikKhxTPx2IqnLGolyHrAA1kCVWPmuCSqsJvLV59jWO/XqpEk+mU05aUzpOgwoVSQtAWdcaqAvVy7ay+7DVXVdYuxvXjsGD52N4ofJPQPVpvs/ZWSmWYHrWinM1V0MptGvyfoxa/tRpgIq9tgiy5S71zIelcSTd3QjQoNi/J9RttoohDok0MpYOCiyIOAReS+0GxTdYbO4N9d0EUxKKki45ZTJ9b84QgzJ6NRCdE3YILwSOJbfliYtEhnRiKQijoMJxyyePuk8jceUu9uri4fl0nI12J7Z+PDEXsyZSlI51uWcuwjKrZ3JFCl2jdA4G+gnkFv1skH8oCq8G+owW6xfuhKKwuCR0pdFoV/kZeFM69UvWSh9LeM9UdGqU/RakreSynTLgkehFKDVW6VxCfxW9WcWNFWz9ZrG3QXT9O6PcooaWS2RAIXiBMucBu9cwG/PWC7ODwjyrENr5sRv7/ve6gWuos6Qi4KTdVPX1CryQQWe0Pl0fg1f5xZzbYZDMRit71ejt5k84KjfUaaX1ZOSFphV+fVrzLGOFiUVUZX6xD7tKBtT7YWjtWeFcXFIdszhq/O7Y3a+LoHdqeWxs/pV4RAuN3lYSnpBv+/bnKviHZresi4eYHixc7nsIdPTRT0yz+eEqfBd7QKddsnogSWrCF1ZsIUbta6L1p+yEj8y5Lafuxfp+GaXh8evKvnz/hL9WeKbZBDugsv93e3izLxQVh4daSj/mx4lPjfAMac6KZqNq8F+WWFooae4591k3WMP/38nYNt3fuyve5bOOwA2/pBsR783vveHs/5liDfHF5fXl72TfqWb81vzXMv12eXezlz7t8QZkhneHzeN0bDkLZe3FyiXOJZHx5fXl+C5+D0UODpA+0PXtFZJIZSqQ88RH1ej2xWuQTlngDaG91HMO++qW3vwX9+mfnTsBf8CFnW40tZBpeVmpKDtBNXJCfwsnUXApF2PNYJppliSFMtv2W7PnMp1WxO8+USoa9GxWOhU7kiWJbujZd+dx0KwTRZintA1Injx77m+6RE7VW2ox+eXwczt1+eXxMJ65RXH37VjHcy25xxpH0K2BqCsjDBc//A6XhpyeJ/ToksV8fHyE2Tp+QWFU3nHJtbOadY1R09sjD64cl6reVz4XKYdibVHcaQg9I7ZLodyP1YXz82bcNFVgVf/dtZVKG+y2hPjTBOvA+rY+QyFe7q5OqBAUpTayebFFNsFWYyEt1pCufodU9vAnbtV1zt96P3h93Bc7cn/IK3PjL0VfgUit3gcaQHDOS44n7iMtSq0deEIuQbsJ5rUVYIJV8G3cfDBLEqtoWroZs+93d8MmwcSGL/iqBq6GqkrICaFn7S7LDSd5md69GEloieFEg48Si2LJC1lykstkDN3zz94D7ibw1nZoBlzAVPJ9tWeJqZCdBta4+qzk+ELGMBXv6g3elYZFW/toJWRW+hoVWp9qTBVAihKmiZepz+pimWCxs74BsNi+99m1zxmJAJ0/pEIvSLqousGHuV62p5+zmqlKfnyuMxxketQukIrDldgLKZbg9eX25ukjTUcfxVb81yvGXcYqZYdz/BQAA//9tt46v" } From 26c79e2d28ebd1886ec8a25413051225da495f4c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 16 Aug 2019 15:11:18 -0600 Subject: [PATCH 3/4] remove calculate_rate from config --- x-pack/metricbeat/module/aws/_meta/config.yml | 1 - x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go | 1 - x-pack/metricbeat/modules.d/aws.yml.disabled | 1 - 3 files changed, 3 deletions(-) diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index f4bd6405ac8a..90dc6dc1af3b 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -4,7 +4,6 @@ - ec2 regions: - us-east-1 - calculate_rate: false - module: aws period: 300s metricsets: diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go index cf990b5dd738..acdf5a901f38 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go @@ -67,7 +67,6 @@ func TestFetch(t *testing.T) { func TestData(t *testing.T) { config, info := mtest.GetConfigForTest("ec2", "300s") - config["calculate_rate"] = true if info != "" { t.Skip("Skipping TestData: " + info) } diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 68ef761800ec..5a3241631b57 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -7,7 +7,6 @@ - ec2 regions: - us-east-1 - calculate_rate: false - module: aws period: 300s metricsets: From ef69a5252b6ffe0c5d1b34e720d148a6c64baf3b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Sat, 17 Aug 2019 10:45:17 -0600 Subject: [PATCH 4/4] Add changelog --- CHANGELOG.next.asciidoc | 1 + x-pack/metricbeat/module/aws/ec2/ec2.go | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1ac104475668..8159aab4c604 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add cgroup memory stats to docker/memory metricset {pull}12916[12916] - Add AWS elb metricset. {pull}12952[12952] {issue}11701[11701] - Add AWS ebs metricset. {pull}13167[13167] {issue}11699[11699] +- Add rate metrics for ec2 metricset. {pull}13203[13203] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 32371ceb163b..b93ebf2e284d 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -255,9 +255,15 @@ func calculateRate(resultMetricsetFields common.MapStr, monitoringState string) period = 60.0 } - metricList := []string{"network.in.bytes", "network.out.bytes", - "network.in.packets", "network.out.packets", "diskio.read.bytes", - "diskio.write.bytes", "diskio.read.count", "diskio.write.count"} + metricList := []string{ + "network.in.bytes", + "network.out.bytes", + "network.in.packets", + "network.out.packets", + "diskio.read.bytes", + "diskio.write.bytes", + "diskio.read.count", + "diskio.write.count"} for _, metricName := range metricList { metricValue, err := resultMetricsetFields.GetValue(metricName)