Cherry-pick #15598 to 7.x: [Metricbeat] Add storage metricset to Goog…

…le Cloud Platform module (#17219)

CHANGELOG.next.asciidoc

@@ -55,7 +55,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - TLS: The behavior of send_certificates and include_raw_certificates options has changed. {pull}15497[15497]
 - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
 - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
-- Redis: fix incorrectly handle with two-words redis command. {issue}14872[14872] {pull}14873[14873] 
 
 *Winlogbeat*
 
@@ -341,6 +340,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release Zookeeper/connection module as GA. {issue}14281[14281] {pull}17043[17043]
 - Replace vpc metricset into vpn, transitgateway and natgateway metricsets. {pull}16892[16892]
 - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833]
+- Add Storage metricsets to GCP module {pull}15598[15598]
 
 *Packetbeat*
 

metricbeat/docs/modules/googlecloud.asciidoc

@@ -101,11 +101,22 @@ metricbeat.modules:
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
+    - loadbalancing
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
+  region: "us-central1"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s
 ----
 
 [float]
@@ -117,7 +128,11 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-googlecloud-loadbalancing,loadbalancing>>
 
+* <<metricbeat-metricset-googlecloud-storage,storage>>
+
 include::googlecloud/compute.asciidoc[]
 
 include::googlecloud/loadbalancing.asciidoc[]
 
+include::googlecloud/storage.asciidoc[]
+

metricbeat/docs/modules/googlecloud/storage.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-googlecloud-storage]]
+=== Google Cloud Platform storage metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-googlecloud,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -109,8 +109,9 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-golang-expvar,expvar>>   
 |<<metricbeat-metricset-golang-heap,heap>>   
 |<<metricbeat-module-googlecloud,Google Cloud Platform>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.2+| .2+|  |<<metricbeat-metricset-googlecloud-compute,compute>> beta[]  
+.3+| .3+|  |<<metricbeat-metricset-googlecloud-compute,compute>> beta[]  
 |<<metricbeat-metricset-googlecloud-loadbalancing,loadbalancing>> beta[]  
+|<<metricbeat-metricset-googlecloud-storage,storage>> beta[]  
 |<<metricbeat-module-graphite,Graphite>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-graphite-server,server>>   
 |<<metricbeat-module-haproxy,HAProxy>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  

x-pack/metricbeat/metricbeat.reference.yml

@@ -484,12 +484,23 @@ metricbeat.modules:
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
+    - loadbalancing
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 300s
 
+- module: googlecloud
+  metricsets:
+    - storage
+  region: "us-central1"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s
+
 #------------------------------- Graphite Module -------------------------------
 - module: graphite
   metricsets: ["server"]

x-pack/metricbeat/module/googlecloud/_meta/config.yml

@@ -1,8 +1,19 @@
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
+    - loadbalancing
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
+  region: "us-central1"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s

x-pack/metricbeat/module/googlecloud/module.yml

@@ -2,3 +2,4 @@ name: googlecloud
 metricsets:
 - compute
 - loadbalancing
+- storage

x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go

@@ -17,7 +17,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error)
 	switch c.ServiceName {
 	case googlecloud.ServiceCompute:
 		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...)
-	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing:
+	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage:
 		return nil, nil
 	default:
 		return nil, errors.Errorf("service '%s' not supported", c.ServiceName)

x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go

@@ -124,6 +124,12 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) {
 	switch service {
 	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing:
 		return
+	case googlecloud.ServiceStorage:
+		if r.config.Region == "" {
+			return
+		}
+
+		f = fmt.Sprintf(`%s AND resource.labels.location = "%s"`, f, r.config.Region)
 	default:
 		if r.config.Region != "" && r.config.Zone != "" {
 			r.logger.Warnf("when region %s and zone %s config parameter "+

x-pack/metricbeat/module/googlecloud/storage/_meta/data.json

@@ -0,0 +1,55 @@
+{
+    "@timestamp": "2020-01-15T19:50:00.000Z",
+    "@metadata": {
+        "beat": "metricbeat",
+        "type": "_doc",
+        "version": "8.0.0"
+    },
+    "ecs": {
+        "version": "1.4.0"
+    },
+    "host": {
+        "name": "mcastro"
+    },
+    "agent": {
+        "type": "metricbeat",
+        "ephemeral_id": "b925f861-f858-4aa8-8075-99fc8e86e736",
+        "hostname": "mcastro",
+        "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228",
+        "version": "8.0.0"
+    },
+    "metricset": {
+        "name": "storage",
+        "period": 300000
+    },
+    "googlecloud": {
+        "labels": {
+            "metrics": {
+                "storage_class": "REGIONAL"
+            },
+            "resource": {
+                "location": "us-central1",
+                "bucket_name": "test-elastic-metricbeat"
+            }
+        },
+        "storage": {
+            "storage": {
+                "object_count": 1
+            }
+        }
+    },
+    "service": {
+        "type": "googlecloud"
+    },
+    "cloud": {
+        "account": {
+            "id": "elastic-metricbeat"
+        },
+        "provider": "googlecloud"
+    },
+    "event": {
+        "dataset": "googlecloud.storage",
+        "module": "googlecloud",
+        "duration": 818254385
+    }
+}

x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc

@@ -0,0 +1,18 @@
+Storage Metricset to fetch metrics from https://cloud.google.com/storage/[Storage] in Google Cloud Platform.
+
+The `storage` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-storage[Stackdriver API]. The field names have been left untouched for people already familiar with them.
+
+You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets.
+
+[float]
+=== Fields
+
+- `storage.api.request_count`: Delta count of API calls, grouped by the API method name and response code.
+- `storage.authz.acl_based_object_access_count`: Delta count of requests that result in an object being granted access solely due to object ACLs.
+- `storage.authz.acl_operations_count`: Usage of ACL operations broken down by type.
+- `storage.authz.object_specific_acl_mutation_count`: Delta count of changes made to object specific ACLs.
+- `storage.network.received_bytes_count`: Delta count of bytes received over the network, grouped by the API method name and response code.
+- `storage.network.sent_bytes_count`: Delta count of bytes sent over the network, grouped by the API method name and response code.
+- `storage.storage.object_count`: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.
+- `storage.storage.total_byte_seconds`: Delta count of bytes received over the network, grouped by the API method name and response code.
+- `storage.storage.total_bytes`: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.

x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml

@@ -0,0 +1,44 @@
+- name: storage
+  release: beta
+  type: group
+  description: Google Cloud Storage metrics
+  fields:
+    - name: api
+      type: group
+      fields:
+      - name: request_count
+        type: long
+        description: Delta count of API calls, grouped by the API method name and response code.
+    - name: authz
+      type: group
+      fields:
+        - name: acl_based_object_access_count
+          type: long
+          description: Delta count of requests that result in an object being granted access solely due to object ACLs.
+        - name: acl_operations_count
+          type: long
+          description: Usage of ACL operations broken down by type.
+        - name: object_specific_acl_mutation_count
+          type: long
+          description: Delta count of changes made to object specific ACLs.
+    - name: network
+      type: group
+      fields:
+        - name: received_bytes_count
+          type: long
+          description: Delta count of bytes received over the network, grouped by the API method name and response code.
+        - name: sent_bytes_count
+          type: long
+          description: Delta count of bytes sent over the network, grouped by the API method name and response code.
+    - name: storage
+      type: group
+      fields:
+        - name: object_count
+          type: long
+          description: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.
+        - name: total_byte_seconds
+          type: long
+          description: Delta count of bytes received over the network, grouped by the API method name and response code.
+        - name: total_bytes
+          type: long
+          description: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.

x-pack/metricbeat/module/googlecloud/storage/manifest.yml

@@ -0,0 +1,17 @@
+default: false
+input:
+  module: googlecloud
+  metricset: stackdriver
+  defaults:
+    stackdriver:
+      service: storage
+      metrics:
+        - "storage.googleapis.com/api/request_count"
+        - "storage.googleapis.com/authz/acl_based_object_access_count"
+        - "storage.googleapis.com/authz/acl_operations_count"
+        - "storage.googleapis.com/authz/object_specific_acl_mutation_count"
+        - "storage.googleapis.com/network/received_bytes_count"
+        - "storage.googleapis.com/network/sent_bytes_count"
+        - "storage.googleapis.com/storage/object_count"
+        - "storage.googleapis.com/storage/total_byte_seconds"
+        - "storage.googleapis.com/storage/total_bytes"

x-pack/metricbeat/modules.d/googlecloud.yml.disabled

@@ -4,8 +4,19 @@
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
+    - loadbalancing
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
+  region: "us-central1"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s