From c840a398b660bb673f614f67253ba389c37fd130 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 24 Mar 2020 17:08:19 +0100 Subject: [PATCH 1/2] Atomic commit --- metricbeat/docs/modules/googlecloud.asciidoc | 12 ++++ .../docs/modules/googlecloud/storage.asciidoc | 23 ++++++++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/metricbeat.reference.yml | 8 +++ .../module/googlecloud/_meta/config.yml | 8 +++ .../metricbeat/module/googlecloud/fields.go | 2 +- .../metricbeat/module/googlecloud/module.yml | 1 + .../stackdriver/metadata_services.go | 2 +- .../stackdriver/metrics_requester.go | 6 ++ .../googlecloud/storage/_meta/data.json | 55 +++++++++++++++++++ .../googlecloud/storage/_meta/docs.asciidoc | 18 ++++++ .../googlecloud/storage/_meta/fields.yml | 44 +++++++++++++++ .../module/googlecloud/storage/manifest.yml | 17 ++++++ .../modules.d/googlecloud.yml.disabled | 8 +++ 14 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 metricbeat/docs/modules/googlecloud/storage.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/data.json create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/googlecloud/storage/manifest.yml diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 9b07841b1ef..19a741eb5d0 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -104,6 +104,14 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" @@ -122,9 +130,13 @@ The following metricsets are available: * <> +* <> + include::googlecloud/compute.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] include::googlecloud/pubsub.asciidoc[] +include::googlecloud/storage.asciidoc[] + diff --git a/metricbeat/docs/modules/googlecloud/storage.asciidoc b/metricbeat/docs/modules/googlecloud/storage.asciidoc new file mode 100644 index 00000000000..86d42459977 --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/storage.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-storage]] +=== Google Cloud Platform storage metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 0365d5b0f2d..a8a6f204203 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -109,9 +109,10 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> beta[] +.4+| .4+| |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index f5a30d37a21..acb13989e5b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -487,6 +487,14 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index b33dfaa03d6..5df057bba18 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -4,6 +4,14 @@ - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 61e4e93847a..3f8bc6b15c1 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsXE1z47gRvftX9M3jlEdzyM2V2qpZJ9mdykzWFXvmqgLBloQYBBigYa3216cAfoiURBKURO3Olm6WRKBfN9DvNT7o9/CKmwdYar2UyKV26Q0ACZL4ALc/hW/h0X8NT5LRQpvs9gbAoERm8QESJHYDkKLlRuQktHqAH24AAH56fIJMp07iDcBCoEztQ/jhPSiWYcvkTLIEpQ0/A9AmxwfQyX+RU/lVs32zD2fRzP5Sf32w7fbpDImljNioFkZwG9/AbixhFv8811nuCBuP74a23c3SaJc3vm0FvjVcj0XPlQuNNrvBbOIRyhJTHFs/dhnv6qzZ4UIYXDMp9x7o67Sv42bnqdF5juk82RDaOddO0cHnK1tSq2XHA61AflJcZ0ItIXRcmYFkA7TCPpf2oeWMvyJNAq7sOhpePedyN8loGLRo3jCdc23Q9vqaapfI3Ul20Nt/uyxBA3oBodfaCGgVvF1pS/5X/3fH5G2jdCSk+I353s8E8cUH3TDuP1VQmJSaM8IUHp++Aq0YgbDAnTGoSG5AKM9dlRNxwC1b4pxE1gVqLO6vvkNYaOPRluEVCixyrdL98atntrCvE80fdv5UfvQ9+VEpUtnbgIXRWZcbO3B0PgkYb7zA8ukX0DmaMB8P50yFZ20E4dTx8UYIFZAeDlABaMoIBQsDIarTI+9IjV4ALeM/63V4LuTkty+wYhYSRAXGKSXU8j4mPRTSWpupMoSjeJtA8PaypLBTZIqPRrdXB9CdX/NqfJXmHYfQoqKpY+dtgH5DMw7VBWLWj6xCIzVLEyaZ4qJl51x16WfNUvixMjCyPF0R5bup151WJ4DoAtIEk/i4qnQuGaHi4mDJ04Lw0bMaGZG4ZqlQNN8AZ5I7GWqGMKnXKyyqA4P/c2gJ1qwcw7LQy43+deOZ2n8owYBTJGTj5/1EqZ4sbFsK89aDMWhzrSzOJuGuEyb2djXG+mu2wR4S51OhSLL+CjW2K513K/dQ2GAgdE1zv/TaibMVY69p8x+/5lqhIsHkj8HbPvPxEGJhNKEsjV7Tar5gnLQZbBVVC+8bsZxFPB3Bw/tdK5fNF0KFCi4qlAN2djmo5IgBcYsvhfyaRtVLr4ayMVvxkS0o5eeXl6cPz0E2oNANLzC6Qmb32aQL+7lR1zjLFWOyqUH5nw8BjwFbkOSFIl0YK0Ndo3+nDXDGV3jnIz3OkYXRirwnxPO5oUPgI0TrPy8vno6tM15atAFkfAVcK4XFOjhBWvvSmUsRnFFpIUhXcbmKy1VcBjv/Q4iLUIRGsXPu3Y6pnf881fPw5O2fsjE0FzFzIqhuRC8xdDe+u37Ki2OB4fyPJ7543oljnaPpbwwBxlPg8SQ4kgbHE+EoKjyRDAdsbXe4YqrsQeBHVtpFKWXjS7594FNAPlBmj4cXVVSfM7CtwrprDVNGvA86aWLywjJWq1JLyna1a6kJPj7+qzl3QKtCtCr/Q1iuinVSd1fFOhlIPBi4KlaErXF6NcWu0GitilWqM24GjQX2+238jNaneHX6DrXpul3U0dV1u2g0hFgYcN0uOminPjX+66xjt+jCx7MVIFwatLaXrHtCF0fUgb0+ff5xuylUUTK8W2gDL49PsJB6bUHQrS1YZ3urUCtgeS4FDxdbwJJBloFWcnO3y3o7TvXfFTjOrdYlgR7HSk3wbnWhFOpSsS8xkW6BnSb2lVeXCf5Bz7qgGaIejR9Q+NZJTrgd4kO3Pcmx4XjHowjh3BfkPqId3gg+cq0/KMNRrYcleEw3ffJ7ntX0kPCed6k5UnKnWXWOE9sRUjtOaKNl9gSRHZBY4vncWjkPFfTvK7KN9JXahhvfNV2ctCZq3vbeElC4Pr1Gg0BoMqHCEqSiqg/Pz5+7jpZHFAPHodwVpG9fGlLpbLil3w/tPMfxNtOaVpgGNn8nFGT2bsvqzaXYrQ38bonx1/vi1D4TyhG2BFGyDZrSu5zZchlaH5kVDl7XZte12XVtNtj5H+Qo/1IUuK1cv32Jo0CF64sJCDc4Wj10jupsAB+LV4Aaxb92ZImp1EeqDdpot1wF6u5AWiHMXWJd0jB9rsvaTy55dsnIS9rWJXV/x9Upz67+4F0JI3ZcfcL46zxDG16dOn3WP7rMSUbiDQuZ8sNX9m69KaXXEtNlIbkft5/r3ef7wvfigRSleEOzCQD6bx5KvSxS97QdZ01MFhdWrPgNq+LBqRb02qF3bPY6Y7MKQP3DHQgFrDXMPcntsnljhleDca4sr8GWwcSwV9EGd2shd3YFqNJcC0X3kDgCpQk2SK1h63fDqdrIud2YdAi0TNHSPNjYQp+z5WlvS31cYig1yxei7qrpVJjrcKnLo1EO5U7KeTOz69fDJsrxhie1I9tX0lpJbdA6STP4pzbAIMUg5mWlfqipxQbXftiSbSsUHzKdBodTZKkUCrs8j4jZ+Q7YhiJ1kPTK+PTjvNS4emPf9YBOO5ghPEeOol1NDs6ugBFhlh8GB1+VFK8YvLD3xQrYtwkHwAZElkvMUFGx4k012qAJCSO+Cv9YombbGTzrYqlcnXRqJTe+WCMmlAWtsNVgFrZ3m8aMnwLFvjMao42fJ16mluINVastcBb2ppEZyJwkkUsEEhn2nPK2ot132Bsf8b/vbDIEZyrvaxPFVoPgRlciMGqWhBcfL1SdVVd/26lJukpLL8TlcfoxRVpxguALnIuL03Nl+smn659Aqg7EcloqqQ22GQ/WglagtHrvOWbTCq1IxxHijk+Xmhs7nl1qQvyN6xR/OGpaxEawvo5zoSlR3scZHPTtzZtc8ONWvy++6Ynr3u8kWtBWg/SClV8ihV19txR53vcm+yIUVf+979xxCv9dLJw2pMYL6QmbUcP/0er/AQAA//8JtEe4" + return "eJzsXE1z2zgSvedX9C3JlqMc9ubamipHszuTWmfHtXZyVYFgS8QaBLj4sEb59VMASIqURBKUKE08pZstkujXDeC9RgPkB3jGzS2spFxxpFza9A2AYYbjLbz9xf8Kc/czPHBillLlb98AKORINN5Cgoa8AUhRU8UKw6S4hZ/eAAD8Mn+AXKaW4xuAJUOe6lt/4QMIkmPL5IyTBLn2lwHMpsBbkMn/kJryp+bzzTasRjX7W/3zwWe3d+doSEoMGfWEYlTHP6A32mAefz+VeWENNm7fDW27mZWStmj82gp8q7vmoeXKhcYzu8Fs4mFCGyIoti52Ge9qrNngkilcE873buhrtK/hZuOpkkWB6SLZGNQLKq0wB++vbHEpVh03tAL5WVCZM7EC33BlBpINmAz7XNqHVhD6jOYs4Mqmo+HVY66wZ+kNhRrVC6YLKhXqXl9TaRO+O8gOevsfmyeoQC7Bt1obASm8t5nUxl11f3cM3jZKaxhn34lrfSKITy7oilD3XwWFcC4pMZjC/OErmIwYYBqoVQqF4RtgwnFX5UQccE1WuDAs7wI1FvdX1yAspXJoy/AyARqpFOl+/9Ujm+nnM40fMv1UnruWXK+EqexswFLJvMuNHTiyOAsYZzxg+fwbyAKVH4+H50yFZ62YwXPHxxkxKMDI4QAFQOeMkLcwEKJ6ehQdU6MXQMv4r3Lt7/Nz8tsXyIiGBFGAskIwsbqJmR4CzVqqc80QiuzlDIK3N0uCnTBTXDS6vTqAbnrNq/FVmnccQo3CnDt2zgbIF1TjUF0gZv3IKjRckjQhnAjKWnamykvvJUnhU2VgZHqaGVPsTr3uaXUCiC4gTTCJi6tIF5wYFJQdTHlaEO4cqxnFEttMFcLjG6CEU8t9zuAH9TrDkB0o/L9FbWBNyj4sE71Cyd83jqndPyUYsMIw3ri8P1GqO4Ntbfy4dWAU6kIKjbOzcNcJA3u7GiP9OdtgC4l1UyFMsv4MNbYpWXQr91DYYCB0TXO/9dqJsxVjr2nzn78XUqAwjPBP3ts+8/EQYmE0oayUXJtssSTUSDX4VFQuvG9EUxJxdwQP7zctbL5YMuEzuKhQDtjZ5aCSIwbELT4VcmsaUS+9GspGdMVHOlDKr09PDx8fvWxA0A0nMLJCpvfZpAv71KhrnOWKMdnUoNzlQ8BjwAaSvFCkg7Ey1DX6d1IBJTTD9y7S4xxZKimM88TQYqHMIfARovXfpydHx9oqJy1SARKaAZVCYFgHJ2jWLnWmnHlnRBoE6SouV3G5istg4z+EuDBhUAkyZe12TO7818mehwdv/5CNobmIkRNBdSNaiaG78c31U14cCwzP/3jii+edONY5mv7GEGA8BR5PgiNpcDwRjqLCE8lwwNa2whWTZQ8CPzLTDqmUjk/59oGfA/KBNHs8vKikesrAthLrrjVMGfE+6EYawi8sY7UqtaRsV7tW0sDd/N/NsQNSBNGq/PdhuSrWSc1dFetkIPFg4KpYEbbG6dU5qkKjtSpWqSYsBo0F9ucVfkbrU7w6vUJtupaLOpq6lotGQ4iFAddy0UE79a7x32cd1aILb89WgHClUOtesu4JXRxRe/b6fP9pWxSqKBneLaWCp/kDLLlca2DmrQ6ssz1VKAWQouCM+oMtoI1CkoMUfPN+l/V2nOo/K3CcW61DAj2OlZrg3OpCycSlYl9iMrIF9jyxr7y6TPAPetYFTRnTo/EDCt/ayfGnQ1zotjs52m/vOBQ+nPuC3Ee0w4XgI9f6gzIc9fSwBI9ppk9+p1lNDwnvtEvNkZJ7nlXnOLEdIbXjhDZaZk8Q2QGJNbRYaM0XPoP+c0W2MX251P7Ed00XJ62Jmqe9twTkj0+vUSEYVDkTfglSUdXHx8f7rq3lEcnAcSh3Benbl4ZUWu1P6fdDm2Y7XudSmgxTz+bvmIBcv9+yenMp9lZ7fteG0OebsGufM2ENtgSRkw2q0ruC6HIZWm+ZBQeva7Pr2uy6Nhts/AfZyr8UBW4z129f4ihQ4PpiAkIVjlYPWaCYDOA8vALUSP6lNdoQkbpItUEraVeZp+4OpBXCwibaJg3TUx3WfrDJo01GHtLWNqnbOy5PebT1P84V32PH5SeEPi9y1P7VqdNH/dzmlhPDXjDIlOu+snXtTAm55piuguTebf+vq883wfdwQ4qcvaDaeAD9Jw+5XIWpe1rFWRrCw4EVzb5jlTxY0YJeO/SOzJ5nZFYBqC+8ByaAtLq5Z3LbfNEY4VVnTDXLa7BlMNHXKtrg3moorM4ARVpIJswNJNaAkAY2aFrd1u+GFbWRqd04axdInqI2C29jC31BVqe9LXW3Qp9qli9Eva+GUzDX4VKXR6McKizni+bMrl8PO9Mcb3hSO7J9Ja01qRVqy80M/iUVEEjRi3mZqR96VGODaz9uybYVio+5TL3DKZKUM4FdnkfEbLoNtqFIHSS9Mj79OC/Vr87Yq+7Q83amD8+Rvaizs4PTGRBjMC8Og4OvgrNn9F7om7ACds/4DWAFLC845ihMWPGmErXXhIQYmvkPS9RsO4NHGZbK1U6nFHzjkjVDmNAgBbYemPnybtOYckMg1J1RKancOHEytWIvKFrPAiW+No1EQW65YQVHMCzHnl3eVrT7NnvjI/7zTpHBO1N5X5sIpQZGlaxEYNQo8S8+Xig7q47+tqemkdW0dEJcbqcfk6SFHQSX4FxcnB4r0w9uuv4FpOpALM9LJbXBNuPBmpkMhBQfHMdsWqFl6ThC3PHpUmNjx7NLDYh/UJniT0cNi9gI1sdxLjQkyvM4g52+PXlTMHrc6vfJPXriuveVRAvaapBeMPNLONPZq6XIad+b7ItQVP73obPi5L8u5ncbUuWE9IRiVPwXrbazQKr2MneqwthjaHlkZYwULJoS+o+39Pd97CmEn5Ebsu3yu4fPQInPlht97RJndyVHk8nU2/evc9bn9BzZHyZBYk32/USPm7U8vkiIxnQRPqa2IJSinqKavROHOgHwdeMw1n19QpSfcYMEHdmtFBGOogMO0JIj30Bq0eWV5Z138/uezN25tOWPCTwJ33RyfTm/bzATJEo+o1vlrIXv095ktgyvLpCyJaMLhzK3Zio+3ok2zYhwGXpO0mbgKusHItj/qZ3jB1jk53WO9nTn4zq7n2I5ddbBuC/dnOjGwa/JTEUc+8R9Ws+WI3qCQ9u+hN7YvPENayhQlTu9rRCUfgDlROsZPGVMwwvhFoHpxqkrQdG3kJLNjQ+RC119n8IibFoRE6oQmuQFdwTkj1y+EF7tFUlr/KMp6dnOCmfCXScuur5l9TrG+NaRKbZFqh0Rwnndqyycb//BOvaPAAAA//+LJduy" } diff --git a/x-pack/metricbeat/module/googlecloud/module.yml b/x-pack/metricbeat/module/googlecloud/module.yml index 10e35ce5a9f..15246c09f3c 100644 --- a/x-pack/metricbeat/module/googlecloud/module.yml +++ b/x-pack/metricbeat/module/googlecloud/module.yml @@ -3,3 +3,4 @@ metricsets: - compute - pubsub - loadbalancing +- storage diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go index 91b7dbbaffa..d628ee4b09d 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go @@ -17,7 +17,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error) switch c.ServiceName { case googlecloud.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...) - case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing: + case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage: return nil, nil default: return nil, errors.Errorf("service '%s' not supported", c.ServiceName) diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 1e05f65e04a..8d2147d285c 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -124,6 +124,12 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { switch service { case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing: return + case googlecloud.ServiceStorage: + if r.config.Region == "" { + return + } + + f = fmt.Sprintf(`%s AND resource.labels.location = "%s"`, f, r.config.Region) default: if r.config.Region != "" && r.config.Zone != "" { r.logger.Warnf("when region %s and zone %s config parameter "+ diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json new file mode 100644 index 00000000000..773f947a41f --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json @@ -0,0 +1,55 @@ +{ + "@timestamp": "2020-01-15T19:50:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "ecs": { + "version": "1.4.0" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "type": "metricbeat", + "ephemeral_id": "b925f861-f858-4aa8-8075-99fc8e86e736", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", + "version": "8.0.0" + }, + "metricset": { + "name": "storage", + "period": 300000 + }, + "googlecloud": { + "labels": { + "metrics": { + "storage_class": "REGIONAL" + }, + "resource": { + "location": "us-central1", + "bucket_name": "test-elastic-metricbeat" + } + }, + "storage": { + "storage": { + "object_count": 1 + } + } + }, + "service": { + "type": "googlecloud" + }, + "cloud": { + "account": { + "id": "elastic-metricbeat" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.storage", + "module": "googlecloud", + "duration": 818254385 + } +} diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc new file mode 100644 index 00000000000..4c9ff62e4ae --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc @@ -0,0 +1,18 @@ +Storage Metricset to fetch metrics from https://cloud.google.com/storage/[Storage] in Google Cloud Platform. + +The `storage` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-storage[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets. + +[float] +=== Fields + +- `storage.api.request_count`: Delta count of API calls, grouped by the API method name and response code. +- `storage.authz.acl_based_object_access_count`: Delta count of requests that result in an object being granted access solely due to object ACLs. +- `storage.authz.acl_operations_count`: Usage of ACL operations broken down by type. +- `storage.authz.object_specific_acl_mutation_count`: Delta count of changes made to object specific ACLs. +- `storage.network.received_bytes_count`: Delta count of bytes received over the network, grouped by the API method name and response code. +- `storage.network.sent_bytes_count`: Delta count of bytes sent over the network, grouped by the API method name and response code. +- `storage.storage.object_count`: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. +- `storage.storage.total_byte_seconds`: Delta count of bytes received over the network, grouped by the API method name and response code. +- `storage.storage.total_bytes`: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml new file mode 100644 index 00000000000..71fcb2bdbeb --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml @@ -0,0 +1,44 @@ +- name: storage + release: beta + type: group + description: Google Cloud Storage metrics + fields: + - name: api + type: group + fields: + - name: request_count + type: long + description: Delta count of API calls, grouped by the API method name and response code. + - name: authz + type: group + fields: + - name: acl_based_object_access_count + type: long + description: Delta count of requests that result in an object being granted access solely due to object ACLs. + - name: acl_operations_count + type: long + description: Usage of ACL operations broken down by type. + - name: object_specific_acl_mutation_count + type: long + description: Delta count of changes made to object specific ACLs. + - name: network + type: group + fields: + - name: received_bytes_count + type: long + description: Delta count of bytes received over the network, grouped by the API method name and response code. + - name: sent_bytes_count + type: long + description: Delta count of bytes sent over the network, grouped by the API method name and response code. + - name: storage + type: group + fields: + - name: object_count + type: long + description: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. + - name: total_byte_seconds + type: long + description: Delta count of bytes received over the network, grouped by the API method name and response code. + - name: total_bytes + type: long + description: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/storage/manifest.yml b/x-pack/metricbeat/module/googlecloud/storage/manifest.yml new file mode 100644 index 00000000000..f462867dcb5 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/manifest.yml @@ -0,0 +1,17 @@ +default: false +input: + module: googlecloud + metricset: stackdriver + defaults: + stackdriver: + service: storage + metrics: + - "storage.googleapis.com/api/request_count" + - "storage.googleapis.com/authz/acl_based_object_access_count" + - "storage.googleapis.com/authz/acl_operations_count" + - "storage.googleapis.com/authz/object_specific_acl_mutation_count" + - "storage.googleapis.com/network/received_bytes_count" + - "storage.googleapis.com/network/sent_bytes_count" + - "storage.googleapis.com/storage/object_count" + - "storage.googleapis.com/storage/total_byte_seconds" + - "storage.googleapis.com/storage/total_bytes" diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 15304fdba00..fc7d792dadf 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -7,6 +7,14 @@ - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" From 4406e09b6c22b52479b05c10f3f60b362d95dddd Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 24 Mar 2020 17:13:42 +0100 Subject: [PATCH 2/2] Add Changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 12f7d2079e1..a1ce267b613 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -264,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Replace vpc metricset into vpn, transitgateway and natgateway metricsets. {pull}16892[16892] - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833] - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119] +- Add Storage metricsets to GCP module {pull}15598[15598] *Packetbeat*