[Metricbeat] Add storage metricset to Google Cloud Platform module

CHANGELOG.next.asciidoc

@@ -264,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Replace vpc metricset into vpn, transitgateway and natgateway metricsets. {pull}16892[16892]
 - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833]
 - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119]
+- Add Storage metricsets to GCP module {pull}15598[15598]
 
 *Packetbeat*
 

metricbeat/docs/modules/googlecloud.asciidoc

@@ -104,6 +104,14 @@ metricbeat.modules:
     - pubsub
     - loadbalancing
   zone: "us-central1-a"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
   region: "us-central1"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
@@ -122,9 +130,13 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-googlecloud-pubsub,pubsub>>
 
+* <<metricbeat-metricset-googlecloud-storage,storage>>
+
 include::googlecloud/compute.asciidoc[]
 
 include::googlecloud/loadbalancing.asciidoc[]
 
 include::googlecloud/pubsub.asciidoc[]
 
+include::googlecloud/storage.asciidoc[]
+

metricbeat/docs/modules/googlecloud/storage.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-googlecloud-storage]]
+=== Google Cloud Platform storage metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-googlecloud,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -109,9 +109,10 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-golang-expvar,expvar>>   
 |<<metricbeat-metricset-golang-heap,heap>>   
 |<<metricbeat-module-googlecloud,Google Cloud Platform>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.3+| .3+|  |<<metricbeat-metricset-googlecloud-compute,compute>> beta[]  
+.4+| .4+|  |<<metricbeat-metricset-googlecloud-compute,compute>> beta[]  
 |<<metricbeat-metricset-googlecloud-loadbalancing,loadbalancing>> beta[]  
 |<<metricbeat-metricset-googlecloud-pubsub,pubsub>> beta[]  
+|<<metricbeat-metricset-googlecloud-storage,storage>> beta[]  
 |<<metricbeat-module-graphite,Graphite>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-graphite-server,server>>   
 |<<metricbeat-module-haproxy,HAProxy>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  

x-pack/metricbeat/metricbeat.reference.yml

@@ -487,6 +487,14 @@ metricbeat.modules:
     - pubsub
     - loadbalancing
   zone: "us-central1-a"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
   region: "us-central1"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"

x-pack/metricbeat/module/googlecloud/_meta/config.yml

@@ -4,6 +4,14 @@
     - pubsub
     - loadbalancing
   zone: "us-central1-a"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
   region: "us-central1"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"

x-pack/metricbeat/module/googlecloud/module.yml

@@ -3,3 +3,4 @@ metricsets:
 - compute
 - pubsub
 - loadbalancing
+- storage

x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go

@@ -17,7 +17,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error)
 	switch c.ServiceName {
 	case googlecloud.ServiceCompute:
 		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...)
-	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing:
+	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage:
 		return nil, nil
 	default:
 		return nil, errors.Errorf("service '%s' not supported", c.ServiceName)

x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go

@@ -124,6 +124,12 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) {
 	switch service {
 	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing:
 		return
+	case googlecloud.ServiceStorage:
+		if r.config.Region == "" {
+			return
+		}
+
+		f = fmt.Sprintf(`%s AND resource.labels.location = "%s"`, f, r.config.Region)
 	default:
 		if r.config.Region != "" && r.config.Zone != "" {
 			r.logger.Warnf("when region %s and zone %s config parameter "+

x-pack/metricbeat/module/googlecloud/storage/_meta/data.json

@@ -0,0 +1,55 @@
+{
+    "@timestamp": "2020-01-15T19:50:00.000Z",
+    "@metadata": {
+        "beat": "metricbeat",
+        "type": "_doc",
+        "version": "8.0.0"
+    },
+    "ecs": {
+        "version": "1.4.0"
+    },
+    "host": {
+        "name": "mcastro"
+    },
+    "agent": {
+        "type": "metricbeat",
+        "ephemeral_id": "b925f861-f858-4aa8-8075-99fc8e86e736",
+        "hostname": "mcastro",
+        "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228",
+        "version": "8.0.0"
+    },
+    "metricset": {
+        "name": "storage",
+        "period": 300000
+    },
+    "googlecloud": {
+        "labels": {
+            "metrics": {
+                "storage_class": "REGIONAL"
+            },
+            "resource": {
+                "location": "us-central1",
+                "bucket_name": "test-elastic-metricbeat"
+            }
+        },
+        "storage": {
+            "storage": {
+                "object_count": 1
+            }
+        }
+    },
+    "service": {
+        "type": "googlecloud"
+    },
+    "cloud": {
+        "account": {
+            "id": "elastic-metricbeat"
+        },
+        "provider": "googlecloud"
+    },
+    "event": {
+        "dataset": "googlecloud.storage",
+        "module": "googlecloud",
+        "duration": 818254385
+    }
+}

x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc

@@ -0,0 +1,18 @@
+Storage Metricset to fetch metrics from https://cloud.google.com/storage/[Storage] in Google Cloud Platform.
+
+The `storage` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-storage[Stackdriver API]. The field names have been left untouched for people already familiar with them.
+
+You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets.
+
+[float]
+=== Fields
+
+- `storage.api.request_count`: Delta count of API calls, grouped by the API method name and response code.
+- `storage.authz.acl_based_object_access_count`: Delta count of requests that result in an object being granted access solely due to object ACLs.
+- `storage.authz.acl_operations_count`: Usage of ACL operations broken down by type.
+- `storage.authz.object_specific_acl_mutation_count`: Delta count of changes made to object specific ACLs.
+- `storage.network.received_bytes_count`: Delta count of bytes received over the network, grouped by the API method name and response code.
+- `storage.network.sent_bytes_count`: Delta count of bytes sent over the network, grouped by the API method name and response code.
+- `storage.storage.object_count`: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.
+- `storage.storage.total_byte_seconds`: Delta count of bytes received over the network, grouped by the API method name and response code.
+- `storage.storage.total_bytes`: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.

x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml

@@ -0,0 +1,44 @@
+- name: storage
+  release: beta
+  type: group
+  description: Google Cloud Storage metrics
+  fields:
+    - name: api
+      type: group
+      fields:
+      - name: request_count
+        type: long
+        description: Delta count of API calls, grouped by the API method name and response code.
+    - name: authz
+      type: group
+      fields:
+        - name: acl_based_object_access_count
+          type: long
+          description: Delta count of requests that result in an object being granted access solely due to object ACLs.
+        - name: acl_operations_count
+          type: long
+          description: Usage of ACL operations broken down by type.
+        - name: object_specific_acl_mutation_count
+          type: long
+          description: Delta count of changes made to object specific ACLs.
+    - name: network
+      type: group
+      fields:
+        - name: received_bytes_count
+          type: long
+          description: Delta count of bytes received over the network, grouped by the API method name and response code.
+        - name: sent_bytes_count
+          type: long
+          description: Delta count of bytes sent over the network, grouped by the API method name and response code.
+    - name: storage
+      type: group
+      fields:
+        - name: object_count
+          type: long
+          description: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.
+        - name: total_byte_seconds
+          type: long
+          description: Delta count of bytes received over the network, grouped by the API method name and response code.
+        - name: total_bytes
+          type: long
+          description: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day.

x-pack/metricbeat/module/googlecloud/storage/manifest.yml

@@ -0,0 +1,17 @@
+default: false
+input:
+  module: googlecloud
+  metricset: stackdriver
+  defaults:
+    stackdriver:
+      service: storage
+      metrics:
+        - "storage.googleapis.com/api/request_count"
+        - "storage.googleapis.com/authz/acl_based_object_access_count"
+        - "storage.googleapis.com/authz/acl_operations_count"
+        - "storage.googleapis.com/authz/object_specific_acl_mutation_count"
+        - "storage.googleapis.com/network/received_bytes_count"
+        - "storage.googleapis.com/network/sent_bytes_count"
+        - "storage.googleapis.com/storage/object_count"
+        - "storage.googleapis.com/storage/total_byte_seconds"
+        - "storage.googleapis.com/storage/total_bytes"

x-pack/metricbeat/modules.d/googlecloud.yml.disabled

@@ -7,6 +7,14 @@
     - pubsub
     - loadbalancing
   zone: "us-central1-a"
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 300s
+
+- module: googlecloud
+  metricsets:
+    - storage
   region: "us-central1"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"