From 52b9330481cb4d59c3282d7a9e41a4db4f4944c0 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Tue, 24 Mar 2020 17:50:54 +0100 Subject: [PATCH] [Metricbeat] Add storage metricset to Google Cloud Platform module (#15598) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/googlecloud.asciidoc | 12 ++++ .../docs/modules/googlecloud/storage.asciidoc | 23 ++++++++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/metricbeat.reference.yml | 8 +++ .../module/googlecloud/_meta/config.yml | 8 +++ .../metricbeat/module/googlecloud/fields.go | 2 +- .../metricbeat/module/googlecloud/module.yml | 1 + .../stackdriver/metadata_services.go | 2 +- .../stackdriver/metrics_requester.go | 6 ++ .../googlecloud/storage/_meta/data.json | 55 +++++++++++++++++++ .../googlecloud/storage/_meta/docs.asciidoc | 18 ++++++ .../googlecloud/storage/_meta/fields.yml | 44 +++++++++++++++ .../module/googlecloud/storage/manifest.yml | 17 ++++++ .../modules.d/googlecloud.yml.disabled | 8 +++ 15 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 metricbeat/docs/modules/googlecloud/storage.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/data.json create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/googlecloud/storage/manifest.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 12f7d2079e1a..a1ce267b6130 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -264,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Replace vpc metricset into vpn, transitgateway and natgateway metricsets. {pull}16892[16892] - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833] - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119] +- Add Storage metricsets to GCP module {pull}15598[15598] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 9b07841b1ef0..19a741eb5d0b 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -104,6 +104,14 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" @@ -122,9 +130,13 @@ The following metricsets are available: * <> +* <> + include::googlecloud/compute.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] include::googlecloud/pubsub.asciidoc[] +include::googlecloud/storage.asciidoc[] + diff --git a/metricbeat/docs/modules/googlecloud/storage.asciidoc b/metricbeat/docs/modules/googlecloud/storage.asciidoc new file mode 100644 index 000000000000..86d424599775 --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/storage.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-storage]] +=== Google Cloud Platform storage metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/googlecloud/storage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 0365d5b0f2d3..a8a6f204203f 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -109,9 +109,10 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> beta[] +.4+| .4+| |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index f5a30d37a213..acb13989e5b1 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -487,6 +487,14 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index b33dfaa03d68..5df057bba18f 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -4,6 +4,14 @@ - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 61e4e93847a8..3f8bc6b15c19 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsXE1z47gRvftX9M3jlEdzyM2V2qpZJ9mdykzWFXvmqgLBloQYBBigYa3216cAfoiURBKURO3Olm6WRKBfN9DvNT7o9/CKmwdYar2UyKV26Q0ACZL4ALc/hW/h0X8NT5LRQpvs9gbAoERm8QESJHYDkKLlRuQktHqAH24AAH56fIJMp07iDcBCoEztQ/jhPSiWYcvkTLIEpQ0/A9AmxwfQyX+RU/lVs32zD2fRzP5Sf32w7fbpDImljNioFkZwG9/AbixhFv8811nuCBuP74a23c3SaJc3vm0FvjVcj0XPlQuNNrvBbOIRyhJTHFs/dhnv6qzZ4UIYXDMp9x7o67Sv42bnqdF5juk82RDaOddO0cHnK1tSq2XHA61AflJcZ0ItIXRcmYFkA7TCPpf2oeWMvyJNAq7sOhpePedyN8loGLRo3jCdc23Q9vqaapfI3Ul20Nt/uyxBA3oBodfaCGgVvF1pS/5X/3fH5G2jdCSk+I353s8E8cUH3TDuP1VQmJSaM8IUHp++Aq0YgbDAnTGoSG5AKM9dlRNxwC1b4pxE1gVqLO6vvkNYaOPRluEVCixyrdL98atntrCvE80fdv5UfvQ9+VEpUtnbgIXRWZcbO3B0PgkYb7zA8ukX0DmaMB8P50yFZ20E4dTx8UYIFZAeDlABaMoIBQsDIarTI+9IjV4ALeM/63V4LuTkty+wYhYSRAXGKSXU8j4mPRTSWpupMoSjeJtA8PaypLBTZIqPRrdXB9CdX/NqfJXmHYfQoqKpY+dtgH5DMw7VBWLWj6xCIzVLEyaZ4qJl51x16WfNUvixMjCyPF0R5bup151WJ4DoAtIEk/i4qnQuGaHi4mDJ04Lw0bMaGZG4ZqlQNN8AZ5I7GWqGMKnXKyyqA4P/c2gJ1qwcw7LQy43+deOZ2n8owYBTJGTj5/1EqZ4sbFsK89aDMWhzrSzOJuGuEyb2djXG+mu2wR4S51OhSLL+CjW2K513K/dQ2GAgdE1zv/TaibMVY69p8x+/5lqhIsHkj8HbPvPxEGJhNKEsjV7Tar5gnLQZbBVVC+8bsZxFPB3Bw/tdK5fNF0KFCi4qlAN2djmo5IgBcYsvhfyaRtVLr4ayMVvxkS0o5eeXl6cPz0E2oNANLzC6Qmb32aQL+7lR1zjLFWOyqUH5nw8BjwFbkOSFIl0YK0Ndo3+nDXDGV3jnIz3OkYXRirwnxPO5oUPgI0TrPy8vno6tM15atAFkfAVcK4XFOjhBWvvSmUsRnFFpIUhXcbmKy1VcBjv/Q4iLUIRGsXPu3Y6pnf881fPw5O2fsjE0FzFzIqhuRC8xdDe+u37Ki2OB4fyPJ7543oljnaPpbwwBxlPg8SQ4kgbHE+EoKjyRDAdsbXe4YqrsQeBHVtpFKWXjS7594FNAPlBmj4cXVVSfM7CtwrprDVNGvA86aWLywjJWq1JLyna1a6kJPj7+qzl3QKtCtCr/Q1iuinVSd1fFOhlIPBi4KlaErXF6NcWu0GitilWqM24GjQX2+238jNaneHX6DrXpul3U0dV1u2g0hFgYcN0uOminPjX+66xjt+jCx7MVIFwatLaXrHtCF0fUgb0+ff5xuylUUTK8W2gDL49PsJB6bUHQrS1YZ3urUCtgeS4FDxdbwJJBloFWcnO3y3o7TvXfFTjOrdYlgR7HSk3wbnWhFOpSsS8xkW6BnSb2lVeXCf5Bz7qgGaIejR9Q+NZJTrgd4kO3Pcmx4XjHowjh3BfkPqId3gg+cq0/KMNRrYcleEw3ffJ7ntX0kPCed6k5UnKnWXWOE9sRUjtOaKNl9gSRHZBY4vncWjkPFfTvK7KN9JXahhvfNV2ctCZq3vbeElC4Pr1Gg0BoMqHCEqSiqg/Pz5+7jpZHFAPHodwVpG9fGlLpbLil3w/tPMfxNtOaVpgGNn8nFGT2bsvqzaXYrQ38bonx1/vi1D4TyhG2BFGyDZrSu5zZchlaH5kVDl7XZte12XVtNtj5H+Qo/1IUuK1cv32Jo0CF64sJCDc4Wj10jupsAB+LV4Aaxb92ZImp1EeqDdpot1wF6u5AWiHMXWJd0jB9rsvaTy55dsnIS9rWJXV/x9Upz67+4F0JI3ZcfcL46zxDG16dOn3WP7rMSUbiDQuZ8sNX9m69KaXXEtNlIbkft5/r3ef7wvfigRSleEOzCQD6bx5KvSxS97QdZ01MFhdWrPgNq+LBqRb02qF3bPY6Y7MKQP3DHQgFrDXMPcntsnljhleDca4sr8GWwcSwV9EGd2shd3YFqNJcC0X3kDgCpQk2SK1h63fDqdrIud2YdAi0TNHSPNjYQp+z5WlvS31cYig1yxei7qrpVJjrcKnLo1EO5U7KeTOz69fDJsrxhie1I9tX0lpJbdA6STP4pzbAIMUg5mWlfqipxQbXftiSbSsUHzKdBodTZKkUCrs8j4jZ+Q7YhiJ1kPTK+PTjvNS4emPf9YBOO5ghPEeOol1NDs6ugBFhlh8GB1+VFK8YvLD3xQrYtwkHwAZElkvMUFGx4k012qAJCSO+Cv9YombbGTzrYqlcnXRqJTe+WCMmlAWtsNVgFrZ3m8aMnwLFvjMao42fJ16mluINVastcBb2ppEZyJwkkUsEEhn2nPK2ot132Bsf8b/vbDIEZyrvaxPFVoPgRlciMGqWhBcfL1SdVVd/26lJukpLL8TlcfoxRVpxguALnIuL03Nl+smn659Aqg7EcloqqQ22GQ/WglagtHrvOWbTCq1IxxHijk+Xmhs7nl1qQvyN6xR/OGpaxEawvo5zoSlR3scZHPTtzZtc8ONWvy++6Ynr3u8kWtBWg/SClV8ihV19txR53vcm+yIUVf+979xxCv9dLJw2pMYL6QmbUcP/0er/AQAA//8JtEe4" + return "eJzsXE1z2zgSvedX9C3JlqMc9ubamipHszuTWmfHtXZyVYFgS8QaBLj4sEb59VMASIqURBKUKE08pZstkujXDeC9RgPkB3jGzS2spFxxpFza9A2AYYbjLbz9xf8Kc/czPHBillLlb98AKORINN5Cgoa8AUhRU8UKw6S4hZ/eAAD8Mn+AXKaW4xuAJUOe6lt/4QMIkmPL5IyTBLn2lwHMpsBbkMn/kJryp+bzzTasRjX7W/3zwWe3d+doSEoMGfWEYlTHP6A32mAefz+VeWENNm7fDW27mZWStmj82gp8q7vmoeXKhcYzu8Fs4mFCGyIoti52Ge9qrNngkilcE873buhrtK/hZuOpkkWB6SLZGNQLKq0wB++vbHEpVh03tAL5WVCZM7EC33BlBpINmAz7XNqHVhD6jOYs4Mqmo+HVY66wZ+kNhRrVC6YLKhXqXl9TaRO+O8gOevsfmyeoQC7Bt1obASm8t5nUxl11f3cM3jZKaxhn34lrfSKITy7oilD3XwWFcC4pMZjC/OErmIwYYBqoVQqF4RtgwnFX5UQccE1WuDAs7wI1FvdX1yAspXJoy/AyARqpFOl+/9Ujm+nnM40fMv1UnruWXK+EqexswFLJvMuNHTiyOAsYZzxg+fwbyAKVH4+H50yFZ62YwXPHxxkxKMDI4QAFQOeMkLcwEKJ6ehQdU6MXQMv4r3Lt7/Nz8tsXyIiGBFGAskIwsbqJmR4CzVqqc80QiuzlDIK3N0uCnTBTXDS6vTqAbnrNq/FVmnccQo3CnDt2zgbIF1TjUF0gZv3IKjRckjQhnAjKWnamykvvJUnhU2VgZHqaGVPsTr3uaXUCiC4gTTCJi6tIF5wYFJQdTHlaEO4cqxnFEttMFcLjG6CEU8t9zuAH9TrDkB0o/L9FbWBNyj4sE71Cyd83jqndPyUYsMIw3ri8P1GqO4Ntbfy4dWAU6kIKjbOzcNcJA3u7GiP9OdtgC4l1UyFMsv4MNbYpWXQr91DYYCB0TXO/9dqJsxVjr2nzn78XUqAwjPBP3ts+8/EQYmE0oayUXJtssSTUSDX4VFQuvG9EUxJxdwQP7zctbL5YMuEzuKhQDtjZ5aCSIwbELT4VcmsaUS+9GspGdMVHOlDKr09PDx8fvWxA0A0nMLJCpvfZpAv71KhrnOWKMdnUoNzlQ8BjwAaSvFCkg7Ey1DX6d1IBJTTD9y7S4xxZKimM88TQYqHMIfARovXfpydHx9oqJy1SARKaAZVCYFgHJ2jWLnWmnHlnRBoE6SouV3G5istg4z+EuDBhUAkyZe12TO7818mehwdv/5CNobmIkRNBdSNaiaG78c31U14cCwzP/3jii+edONY5mv7GEGA8BR5PgiNpcDwRjqLCE8lwwNa2whWTZQ8CPzLTDqmUjk/59oGfA/KBNHs8vKikesrAthLrrjVMGfE+6EYawi8sY7UqtaRsV7tW0sDd/N/NsQNSBNGq/PdhuSrWSc1dFetkIPFg4KpYEbbG6dU5qkKjtSpWqSYsBo0F9ucVfkbrU7w6vUJtupaLOpq6lotGQ4iFAddy0UE79a7x32cd1aILb89WgHClUOtesu4JXRxRe/b6fP9pWxSqKBneLaWCp/kDLLlca2DmrQ6ssz1VKAWQouCM+oMtoI1CkoMUfPN+l/V2nOo/K3CcW61DAj2OlZrg3OpCycSlYl9iMrIF9jyxr7y6TPAPetYFTRnTo/EDCt/ayfGnQ1zotjs52m/vOBQ+nPuC3Ee0w4XgI9f6gzIc9fSwBI9ppk9+p1lNDwnvtEvNkZJ7nlXnOLEdIbXjhDZaZk8Q2QGJNbRYaM0XPoP+c0W2MX251P7Ed00XJ62Jmqe9twTkj0+vUSEYVDkTfglSUdXHx8f7rq3lEcnAcSh3Benbl4ZUWu1P6fdDm2Y7XudSmgxTz+bvmIBcv9+yenMp9lZ7fteG0OebsGufM2ENtgSRkw2q0ruC6HIZWm+ZBQeva7Pr2uy6Nhts/AfZyr8UBW4z129f4ihQ4PpiAkIVjlYPWaCYDOA8vALUSP6lNdoQkbpItUEraVeZp+4OpBXCwibaJg3TUx3WfrDJo01GHtLWNqnbOy5PebT1P84V32PH5SeEPi9y1P7VqdNH/dzmlhPDXjDIlOu+snXtTAm55piuguTebf+vq883wfdwQ4qcvaDaeAD9Jw+5XIWpe1rFWRrCw4EVzb5jlTxY0YJeO/SOzJ5nZFYBqC+8ByaAtLq5Z3LbfNEY4VVnTDXLa7BlMNHXKtrg3moorM4ARVpIJswNJNaAkAY2aFrd1u+GFbWRqd04axdInqI2C29jC31BVqe9LXW3Qp9qli9Eva+GUzDX4VKXR6McKizni+bMrl8PO9Mcb3hSO7J9Ja01qRVqy80M/iUVEEjRi3mZqR96VGODaz9uybYVio+5TL3DKZKUM4FdnkfEbLoNtqFIHSS9Mj79OC/Vr87Yq+7Q83amD8+Rvaizs4PTGRBjMC8Og4OvgrNn9F7om7ACds/4DWAFLC845ihMWPGmErXXhIQYmvkPS9RsO4NHGZbK1U6nFHzjkjVDmNAgBbYemPnybtOYckMg1J1RKancOHEytWIvKFrPAiW+No1EQW65YQVHMCzHnl3eVrT7NnvjI/7zTpHBO1N5X5sIpQZGlaxEYNQo8S8+Xig7q47+tqemkdW0dEJcbqcfk6SFHQSX4FxcnB4r0w9uuv4FpOpALM9LJbXBNuPBmpkMhBQfHMdsWqFl6ThC3PHpUmNjx7NLDYh/UJniT0cNi9gI1sdxLjQkyvM4g52+PXlTMHrc6vfJPXriuveVRAvaapBeMPNLONPZq6XIad+b7ItQVP73obPi5L8u5ncbUuWE9IRiVPwXrbazQKr2MneqwthjaHlkZYwULJoS+o+39Pd97CmEn5Ebsu3yu4fPQInPlht97RJndyVHk8nU2/evc9bn9BzZHyZBYk32/USPm7U8vkiIxnQRPqa2IJSinqKavROHOgHwdeMw1n19QpSfcYMEHdmtFBGOogMO0JIj30Bq0eWV5Z138/uezN25tOWPCTwJ33RyfTm/bzATJEo+o1vlrIXv095ktgyvLpCyJaMLhzK3Zio+3ok2zYhwGXpO0mbgKusHItj/qZ3jB1jk53WO9nTn4zq7n2I5ddbBuC/dnOjGwa/JTEUc+8R9Ws+WI3qCQ9u+hN7YvPENayhQlTu9rRCUfgDlROsZPGVMwwvhFoHpxqkrQdG3kJLNjQ+RC119n8IibFoRE6oQmuQFdwTkj1y+EF7tFUlr/KMp6dnOCmfCXScuur5l9TrG+NaRKbZFqh0Rwnndqyycb//BOvaPAAAA//+LJduy" } diff --git a/x-pack/metricbeat/module/googlecloud/module.yml b/x-pack/metricbeat/module/googlecloud/module.yml index 10e35ce5a9fa..15246c09f3c5 100644 --- a/x-pack/metricbeat/module/googlecloud/module.yml +++ b/x-pack/metricbeat/module/googlecloud/module.yml @@ -3,3 +3,4 @@ metricsets: - compute - pubsub - loadbalancing +- storage diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go index 91b7dbbaffad..d628ee4b09d5 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go @@ -17,7 +17,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error) switch c.ServiceName { case googlecloud.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...) - case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing: + case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage: return nil, nil default: return nil, errors.Errorf("service '%s' not supported", c.ServiceName) diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 1e05f65e04a8..8d2147d285c7 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -124,6 +124,12 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { switch service { case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing: return + case googlecloud.ServiceStorage: + if r.config.Region == "" { + return + } + + f = fmt.Sprintf(`%s AND resource.labels.location = "%s"`, f, r.config.Region) default: if r.config.Region != "" && r.config.Zone != "" { r.logger.Warnf("when region %s and zone %s config parameter "+ diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json new file mode 100644 index 000000000000..773f947a41f6 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json @@ -0,0 +1,55 @@ +{ + "@timestamp": "2020-01-15T19:50:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "ecs": { + "version": "1.4.0" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "type": "metricbeat", + "ephemeral_id": "b925f861-f858-4aa8-8075-99fc8e86e736", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", + "version": "8.0.0" + }, + "metricset": { + "name": "storage", + "period": 300000 + }, + "googlecloud": { + "labels": { + "metrics": { + "storage_class": "REGIONAL" + }, + "resource": { + "location": "us-central1", + "bucket_name": "test-elastic-metricbeat" + } + }, + "storage": { + "storage": { + "object_count": 1 + } + } + }, + "service": { + "type": "googlecloud" + }, + "cloud": { + "account": { + "id": "elastic-metricbeat" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.storage", + "module": "googlecloud", + "duration": 818254385 + } +} diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc new file mode 100644 index 000000000000..4c9ff62e4ae8 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/docs.asciidoc @@ -0,0 +1,18 @@ +Storage Metricset to fetch metrics from https://cloud.google.com/storage/[Storage] in Google Cloud Platform. + +The `storage` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-storage[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets. + +[float] +=== Fields + +- `storage.api.request_count`: Delta count of API calls, grouped by the API method name and response code. +- `storage.authz.acl_based_object_access_count`: Delta count of requests that result in an object being granted access solely due to object ACLs. +- `storage.authz.acl_operations_count`: Usage of ACL operations broken down by type. +- `storage.authz.object_specific_acl_mutation_count`: Delta count of changes made to object specific ACLs. +- `storage.network.received_bytes_count`: Delta count of bytes received over the network, grouped by the API method name and response code. +- `storage.network.sent_bytes_count`: Delta count of bytes sent over the network, grouped by the API method name and response code. +- `storage.storage.object_count`: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. +- `storage.storage.total_byte_seconds`: Delta count of bytes received over the network, grouped by the API method name and response code. +- `storage.storage.total_bytes`: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml new file mode 100644 index 000000000000..71fcb2bdbeb9 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/fields.yml @@ -0,0 +1,44 @@ +- name: storage + release: beta + type: group + description: Google Cloud Storage metrics + fields: + - name: api + type: group + fields: + - name: request_count + type: long + description: Delta count of API calls, grouped by the API method name and response code. + - name: authz + type: group + fields: + - name: acl_based_object_access_count + type: long + description: Delta count of requests that result in an object being granted access solely due to object ACLs. + - name: acl_operations_count + type: long + description: Usage of ACL operations broken down by type. + - name: object_specific_acl_mutation_count + type: long + description: Delta count of changes made to object specific ACLs. + - name: network + type: group + fields: + - name: received_bytes_count + type: long + description: Delta count of bytes received over the network, grouped by the API method name and response code. + - name: sent_bytes_count + type: long + description: Delta count of bytes sent over the network, grouped by the API method name and response code. + - name: storage + type: group + fields: + - name: object_count + type: long + description: Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. + - name: total_byte_seconds + type: long + description: Delta count of bytes received over the network, grouped by the API method name and response code. + - name: total_bytes + type: long + description: Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/storage/manifest.yml b/x-pack/metricbeat/module/googlecloud/storage/manifest.yml new file mode 100644 index 000000000000..f462867dcb58 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/manifest.yml @@ -0,0 +1,17 @@ +default: false +input: + module: googlecloud + metricset: stackdriver + defaults: + stackdriver: + service: storage + metrics: + - "storage.googleapis.com/api/request_count" + - "storage.googleapis.com/authz/acl_based_object_access_count" + - "storage.googleapis.com/authz/acl_operations_count" + - "storage.googleapis.com/authz/object_specific_acl_mutation_count" + - "storage.googleapis.com/network/received_bytes_count" + - "storage.googleapis.com/network/sent_bytes_count" + - "storage.googleapis.com/storage/object_count" + - "storage.googleapis.com/storage/total_byte_seconds" + - "storage.googleapis.com/storage/total_bytes" diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 15304fdba006..fc7d792dadf2 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -7,6 +7,14 @@ - pubsub - loadbalancing zone: "us-central1-a" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 300s + +- module: googlecloud + metricsets: + - storage region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path"