From 9500d18966591e0f0f070b1012ff3febb5a5ca06 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 19 Nov 2018 18:28:14 +0100 Subject: [PATCH 01/36] created iis module --- metricbeat/docs/fields.asciidoc | 32 +++++++++ metricbeat/docs/modules/iis.asciidoc | 39 +++++++++++ metricbeat/docs/modules/iis/system.asciidoc | 23 +++++++ metricbeat/docs/modules_list.asciidoc | 3 + metricbeat/include/list.go | 2 + metricbeat/metricbeat.reference.yml | 8 +++ metricbeat/module/iis/_meta/config.yml | 6 ++ metricbeat/module/iis/_meta/docs.asciidoc | 2 + metricbeat/module/iis/_meta/fields.yml | 11 +++ metricbeat/module/iis/doc.go | 2 + metricbeat/module/iis/fields.go | 35 ++++++++++ metricbeat/module/iis/helper.go | 68 +++++++++++++++++++ metricbeat/module/iis/system/_meta/data.json | 19 ++++++ .../module/iis/system/_meta/docs.asciidoc | 1 + metricbeat/module/iis/system/_meta/fields.yml | 9 +++ metricbeat/module/iis/system/data.go | 17 +++++ metricbeat/module/iis/system/system.go | 59 ++++++++++++++++ metricbeat/modules.d/iis.yml | 9 +++ metricbeat/modules.d/system.yml | 2 +- 19 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/iis.asciidoc create mode 100644 metricbeat/docs/modules/iis/system.asciidoc create mode 100644 metricbeat/module/iis/_meta/config.yml create mode 100644 metricbeat/module/iis/_meta/docs.asciidoc create mode 100644 metricbeat/module/iis/_meta/fields.yml create mode 100644 metricbeat/module/iis/doc.go create mode 100644 metricbeat/module/iis/fields.go create mode 100644 metricbeat/module/iis/helper.go create mode 100644 metricbeat/module/iis/system/_meta/data.json create mode 100644 metricbeat/module/iis/system/_meta/docs.asciidoc create mode 100644 metricbeat/module/iis/system/_meta/fields.yml create mode 100644 metricbeat/module/iis/system/data.go create mode 100644 metricbeat/module/iis/system/system.go create mode 100644 metricbeat/modules.d/iis.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b3364abc3222..cc0f7a08fa95 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -31,6 +31,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -8866,6 +8867,37 @@ json metricset server +[[exported-fields-iis]] +== iis fields + +experimental[] +iis module + + + +[float] +== iis fields + + + + +[float] +== system fields + +system + + + +*`iis.system.example`*:: ++ +-- +type: keyword + +Example field + + +-- + [[exported-fields-jolokia]] == Jolokia fields diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc new file mode 100644 index 000000000000..8bef2784c112 --- /dev/null +++ b/metricbeat/docs/modules/iis.asciidoc @@ -0,0 +1,39 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-iis]] +== iis module + +experimental[] + +This is the iis module. + + + +[float] +=== Example configuration + +The iis module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: iis + metricsets: ["system"] + enabled: false + period: 10s + hosts: ["localhost"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::iis/system.asciidoc[] + diff --git a/metricbeat/docs/modules/iis/system.asciidoc b/metricbeat/docs/modules/iis/system.asciidoc new file mode 100644 index 000000000000..be73da63aa20 --- /dev/null +++ b/metricbeat/docs/modules/iis/system.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-iis-system]] +=== iis system metricset + +experimental[] + +include::../../../module/iis/system/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/system/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 0d52dcf865a1..ae48e296663d 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -60,6 +60,8 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> beta[] +|<> experimental[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> experimental[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | @@ -166,6 +168,7 @@ include::modules/golang.asciidoc[] include::modules/graphite.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/http.asciidoc[] +include::modules/iis.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index d241c79389bc..eeb3147bf707 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -81,6 +81,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http" _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" + _ "github.com/elastic/beats/metricbeat/module/iis" + _ "github.com/elastic/beats/metricbeat/module/iis/system" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4db8f49b71f0..de2d7f7cc430 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -296,6 +296,14 @@ metricbeat.modules: # fields: # added to the the response in root. overwrites existing fields # key: "value" +#--------------------------------- iis Module -------------------------------- +- module: iis + metricsets: ["system"] + enabled: true + period: 10s + hosts: ["localhost"] + + #------------------------------- Jolokia Module ------------------------------ - module: jolokia #metricsets: ["jmx"] diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml new file mode 100644 index 000000000000..6f16b2723fc7 --- /dev/null +++ b/metricbeat/module/iis/_meta/config.yml @@ -0,0 +1,6 @@ +- module: iis + metricsets: ["system"] + enabled: true + period: 10s + hosts: ["localhost"] + diff --git a/metricbeat/module/iis/_meta/docs.asciidoc b/metricbeat/module/iis/_meta/docs.asciidoc new file mode 100644 index 000000000000..a59df72134d5 --- /dev/null +++ b/metricbeat/module/iis/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +This is the iis module. + diff --git a/metricbeat/module/iis/_meta/fields.yml b/metricbeat/module/iis/_meta/fields.yml new file mode 100644 index 000000000000..382fd01c9fb3 --- /dev/null +++ b/metricbeat/module/iis/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: iis + title: "iis" + description: > + experimental[] + + iis module + fields: + - name: iis + type: group + description: > + fields: diff --git a/metricbeat/module/iis/doc.go b/metricbeat/module/iis/doc.go new file mode 100644 index 000000000000..889f4527de8d --- /dev/null +++ b/metricbeat/module/iis/doc.go @@ -0,0 +1,2 @@ +// Package iis is a Metricbeat module that contains MetricSets. +package iis diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go new file mode 100644 index 000000000000..f36800e42269 --- /dev/null +++ b/metricbeat/module/iis/fields.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package iis + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "iis", Asset); err != nil { + panic(err) + } +} + +// Asset returns asset data +func Asset() string { + return "eJx8j0EOgjAQRfc9xQ97LtCFO09hXBD7NRNa2rQl0tsbqBpE9C3/JO9lWvQsGiJJAVmypUYjkhoFGKZLlJDFDxoHBQCcAqM4Drmzp7NaNpEE581oqYCr0Jqkl0OLoXN8yWdyCdS4RT+G57LT+JSsRamkTPee93w/nZUvwza1znHqXFi+WlObPcvdR7O5/SnPHKuwRtUjAAD//ybiY20=" +} diff --git a/metricbeat/module/iis/helper.go b/metricbeat/module/iis/helper.go new file mode 100644 index 000000000000..d568ab3e1798 --- /dev/null +++ b/metricbeat/module/iis/helper.go @@ -0,0 +1,68 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package iis + +import ( + "github.com/elastic/beats/metricbeat/module/windows/perfmon" +) + +var ( + + webservice_counters = map[string]string { + "website.bytes_sent_per_sec":"\\Web Service(*)\\Bytes Sent/sec", + "website.total_bytes_sent_per_sec":"\\Web Service(*)\\Total Bytes Sent", + "website.bytes_recv_per_sec":"\\Web Service(*)\\Bytes Received/sec", + "website.total_bytes_recv_per_sec":"\\Web Service(*)\\Total Bytes Received", +} + + + Config = perfmon.Config{ + IgnoreNECounters: true, + CounterConfig: GetPerfCounters("webserver"), + } +) +func GetPerfCounters(metricset string) []perfmon.CounterConfig { + counters = []perfmon.CounterConfig{} + switch metricset { + case "webserver": + for k, v := range webservice_counters { + counter := perfmon.CounterConfig{ + InstanceLabel: "website.name", + MeasurementLabel: k, + Query: v, + Format: "float", + } + counters = append(counters, k, v) + } + case "app_pool": + for k, v := range webservice_counters { + counter := perfmon.CounterConfig{ + InstanceLabel: "website.name", + MeasurementLabel: k, + Query: v, + Format: "float", + } + counters = append(counters, k, v) + } + + } + return counters; +} + diff --git a/metricbeat/module/iis/system/_meta/data.json b/metricbeat/module/iis/system/_meta/data.json new file mode 100644 index 000000000000..b44caf91042a --- /dev/null +++ b/metricbeat/module/iis/system/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"iis", + "name":"system", + "rtt":44269 + }, + "iis":{ + "system":{ + "example": "system" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/iis/system/_meta/docs.asciidoc b/metricbeat/module/iis/system/_meta/docs.asciidoc new file mode 100644 index 000000000000..db94dc7c8e80 --- /dev/null +++ b/metricbeat/module/iis/system/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the system metricset of the module iis. diff --git a/metricbeat/module/iis/system/_meta/fields.yml b/metricbeat/module/iis/system/_meta/fields.yml new file mode 100644 index 000000000000..ecbd7ffc7869 --- /dev/null +++ b/metricbeat/module/iis/system/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: system + type: group + description: > + system + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/iis/system/data.go b/metricbeat/module/iis/system/data.go new file mode 100644 index 000000000000..67787e1915eb --- /dev/null +++ b/metricbeat/module/iis/system/data.go @@ -0,0 +1,17 @@ +package system + +import ( + "github.com/elastic/beats/metricbeat/mb" +) + + + +func eventsMapping(r mb.ReporterV2, events []mb.Event) { + + //remapping process details to match the naming format + for _, event := range events { + + + r.Event(event) + } +} diff --git a/metricbeat/module/iis/system/system.go b/metricbeat/module/iis/system/system.go new file mode 100644 index 000000000000..b0175e983ed1 --- /dev/null +++ b/metricbeat/module/iis/system/system.go @@ -0,0 +1,59 @@ +package system + +import ( + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/iis" + "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "github.com/pkg/errors" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("iis", "system", New, mb.DefaultMetricSet()) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + reader *perfmon.PerfmonReader +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The iis system metricset is experimental.") + + config := iis.Config + //if err := base.Module().UnpackConfig(&config); err != nil { + // return nil, err + //} + + reader, err := perfmon.NewPerfmonReader(config) + if err != nil { + return nil, errors.Wrap(err, "initialization of reader failed") + } + + return &MetricSet{ + BaseMetricSet: base, + reader : reader, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + events, err := m.reader.Read() + if err != nil { + err = errors.Wrap(err, "failed reading counters") + report.Error(err) + } + eventsMapping(report, events) +} diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml new file mode 100644 index 000000000000..1dc41125c69e --- /dev/null +++ b/metricbeat/modules.d/iis.yml @@ -0,0 +1,9 @@ +# Module: iis +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-iis.html + +- module: iis + metricsets: ["system"] + enabled: true + period: 10s + hosts: ["localhost"] + diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index b4d38cc520bc..1b1413b31c35 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -5,7 +5,7 @@ period: 10s metricsets: - cpu - - load + #- load - memory - network - process From bab0cd4003ef0f0c92780aa41e8a5e6e94b437f6 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 27 Mar 2019 16:48:57 +0100 Subject: [PATCH 02/36] work in progress --- metricbeat/docs/fields.asciidoc | 23 ++++- metricbeat/docs/modules/iis.asciidoc | 15 ++-- .../docs/modules/iis/webserver.asciidoc | 23 +++++ .../iis/{system.asciidoc => website.asciidoc} | 8 +- metricbeat/docs/modules/system.asciidoc | 2 +- metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 3 +- metricbeat/metricbeat.reference.yml | 7 +- metricbeat/module/iis/_meta/config.yml | 5 +- metricbeat/module/iis/fields.go | 2 +- metricbeat/module/iis/helper.go | 87 ++++++++++++++++--- .../module/iis/system/_meta/docs.asciidoc | 1 - metricbeat/module/iis/system/data.go | 17 ---- .../module/iis/webserver/_meta/data.json | 19 ++++ .../module/iis/webserver/_meta/docs.asciidoc | 1 + .../module/iis/webserver/_meta/fields.yml | 9 ++ metricbeat/module/iis/webserver/data.go | 44 ++++++++++ metricbeat/module/iis/webserver/webserver.go | 55 ++++++++++++ .../iis/{system => website}/_meta/data.json | 6 +- .../module/iis/website/_meta/docs.asciidoc | 1 + .../iis/{system => website}/_meta/fields.yml | 4 +- metricbeat/module/iis/website/data.go | 40 +++++++++ .../{system/system.go => website/website.go} | 11 +-- .../module/system/_meta/config.reference.yml | 2 +- .../modules.d/{iis.yml => iis.yml.disabled} | 5 +- metricbeat/modules.d/system.yml | 2 +- 26 files changed, 329 insertions(+), 66 deletions(-) create mode 100644 metricbeat/docs/modules/iis/webserver.asciidoc rename metricbeat/docs/modules/iis/{system.asciidoc => website.asciidoc} (61%) delete mode 100644 metricbeat/module/iis/system/_meta/docs.asciidoc delete mode 100644 metricbeat/module/iis/system/data.go create mode 100644 metricbeat/module/iis/webserver/_meta/data.json create mode 100644 metricbeat/module/iis/webserver/_meta/docs.asciidoc create mode 100644 metricbeat/module/iis/webserver/_meta/fields.yml create mode 100644 metricbeat/module/iis/webserver/data.go create mode 100644 metricbeat/module/iis/webserver/webserver.go rename metricbeat/module/iis/{system => website}/_meta/data.json (77%) create mode 100644 metricbeat/module/iis/website/_meta/docs.asciidoc rename metricbeat/module/iis/{system => website}/_meta/fields.yml (81%) create mode 100644 metricbeat/module/iis/website/data.go rename metricbeat/module/iis/{system/system.go => website/website.go} (86%) rename metricbeat/modules.d/{iis.yml => iis.yml.disabled} (75%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index cc0f7a08fa95..923d737df2a6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -8882,13 +8882,30 @@ iis module [float] -== system fields +== webserver fields + +webserver + + + +*`iis.webserver.example`*:: ++ +-- +type: keyword + +Example field + + +-- + +[float] +== website fields -system +website -*`iis.system.example`*:: +*`iis.website.example`*:: + -- type: keyword diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 8bef2784c112..24971b341919 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -21,10 +21,11 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: iis - metricsets: ["system"] - enabled: false + metricsets: + - webserver + - website + enabled: true period: 10s - hosts: ["localhost"] ---- @@ -33,7 +34,11 @@ metricbeat.modules: The following metricsets are available: -* <> +* <> -include::iis/system.asciidoc[] +* <> + +include::iis/webserver.asciidoc[] + +include::iis/website.asciidoc[] diff --git a/metricbeat/docs/modules/iis/webserver.asciidoc b/metricbeat/docs/modules/iis/webserver.asciidoc new file mode 100644 index 000000000000..a9029e7eacaf --- /dev/null +++ b/metricbeat/docs/modules/iis/webserver.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-iis-webserver]] +=== iis webserver metricset + +experimental[] + +include::../../../module/iis/webserver/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/webserver/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/iis/system.asciidoc b/metricbeat/docs/modules/iis/website.asciidoc similarity index 61% rename from metricbeat/docs/modules/iis/system.asciidoc rename to metricbeat/docs/modules/iis/website.asciidoc index be73da63aa20..10deb2093add 100644 --- a/metricbeat/docs/modules/iis/system.asciidoc +++ b/metricbeat/docs/modules/iis/website.asciidoc @@ -2,12 +2,12 @@ This file is generated! See scripts/docs_collector.py //// -[[metricbeat-metricset-iis-system]] -=== iis system metricset +[[metricbeat-metricset-iis-website]] +=== iis website metricset experimental[] -include::../../../module/iis/system/_meta/docs.asciidoc[] +include::../../../module/iis/website/_meta/docs.asciidoc[] ==== Fields @@ -19,5 +19,5 @@ Here is an example document generated by this metricset: [source,json] ---- -include::../../../module/iis/system/_meta/data.json[] +include::../../../module/iis/website/_meta/data.json[] ---- diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index bffa3adc06df..821cccc4c398 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -31,7 +31,7 @@ metricbeat.modules: - module: system metricsets: - cpu # CPU usage - - load # CPU load averages + #- load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index ae48e296663d..4a4265a3da55 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -61,7 +61,8 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> |<> beta[] |<> experimental[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> experimental[] +.2+| .2+| |<> experimental[] +|<> experimental[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index eeb3147bf707..ffee7cbcc804 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -82,7 +82,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" _ "github.com/elastic/beats/metricbeat/module/iis" - _ "github.com/elastic/beats/metricbeat/module/iis/system" + _ "github.com/elastic/beats/metricbeat/module/iis/webserver" + _ "github.com/elastic/beats/metricbeat/module/iis/website" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index de2d7f7cc430..cf41b96b623d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -51,7 +51,7 @@ metricbeat.modules: - module: system metricsets: - cpu # CPU usage - - load # CPU load averages + #- load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics @@ -298,10 +298,11 @@ metricbeat.modules: #--------------------------------- iis Module -------------------------------- - module: iis - metricsets: ["system"] + metricsets: + - webserver + - website enabled: true period: 10s - hosts: ["localhost"] #------------------------------- Jolokia Module ------------------------------ diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml index 6f16b2723fc7..995096a926e8 100644 --- a/metricbeat/module/iis/_meta/config.yml +++ b/metricbeat/module/iis/_meta/config.yml @@ -1,6 +1,7 @@ - module: iis - metricsets: ["system"] + metricsets: + - webserver + - website enabled: true period: 10s - hosts: ["localhost"] diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index f36800e42269..ae0bd93a542a 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJx8j0EOgjAQRfc9xQ97LtCFO09hXBD7NRNa2rQl0tsbqBpE9C3/JO9lWvQsGiJJAVmypUYjkhoFGKZLlJDFDxoHBQCcAqM4Drmzp7NaNpEE581oqYCr0Jqkl0OLoXN8yWdyCdS4RT+G57LT+JSsRamkTPee93w/nZUvwza1znHqXFi+WlObPcvdR7O5/SnPHKuwRtUjAAD//ybiY20=" + return "eJzEj7FugzAYhHc/xYmdF/DQrU9RdaD1tfqFjS3bFHj7CpxEgJwsGXLjnfR9uhY9Fw2RpIAs2VKjEUmNAgzTd5SQxQ8abwoAOAdGcRxyZz8+1daJJDhvRksF/AitSXobWgyd4xW+Ji+BGr/Rj+HSVBxHyB408Ssx/jHelhryLrakBjkL91LOnQvbt32Ktucy+WhO2wP5mvcCLNLqR8l89uER8cJ//wEAAP//Ae+cTA==" } diff --git a/metricbeat/module/iis/helper.go b/metricbeat/module/iis/helper.go index d568ab3e1798..c1cbf065ca5c 100644 --- a/metricbeat/module/iis/helper.go +++ b/metricbeat/module/iis/helper.go @@ -21,48 +21,113 @@ package iis import ( "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "github.com/elastic/beats/libbeat/common" + "errors" ) var ( - webservice_counters = map[string]string { + website_counters = map[string]string { "website.bytes_sent_per_sec":"\\Web Service(*)\\Bytes Sent/sec", "website.total_bytes_sent_per_sec":"\\Web Service(*)\\Total Bytes Sent", "website.bytes_recv_per_sec":"\\Web Service(*)\\Bytes Received/sec", "website.total_bytes_recv_per_sec":"\\Web Service(*)\\Total Bytes Received", + + //\Web Service(*)\Total Files Sent + //\Web Service(*)\Files Sent/sec + //\Web Service(*)\Total Files Received + //\Web Service(*)\Files Received/sec + //\Web Service(*)\Current Connections + //\Web Service(*)\Maximum Connections + //\Web Service(*)\Total Connection Attempts (all instances) + //\Web Service(*)\Total Get Requests + //\Web Service(*)\Get Requests/sec + //\Web Service(*)\Total Post Requests + //\Web Service(*)\Post Requests/sec } + webserver_counters = map[string]string { + "webserver.total_bytes_sent_per_sec":"\\Web Service(_Total)\\Total Bytes Sent", + "webserver.total_bytes_recv_per_sec":"\\Web Service(_Total)\\Total Bytes Received", + //\Web Service(*)\Total Files Sent + //\Web Service(*)\Files Sent/sec + //\Web Service(*)\Total Files Received + //\Web Service(*)\Files Received/sec + //\Web Service(*)\Current Connections + //\Web Service(*)\Maximum Connections + //\Web Service(*)\Total Connection Attempts (all instances) + //\Web Service(*)\Total Get Requests + //\Web Service(*)\Get Requests/sec + //\Web Service(*)\Total Post Requests + //\Web Service(*)\Post Requests/sec + + //cache + //"cache": { + //"file_cache_count": "2", + //"file_cache_memory_usage": "699", + //"file_cache_hits": "18506471", + //"file_cache_misses": "46266060", + //"total_files_cached": "10", + //"output_cache_count": "0", + //"output_cache_memory_usage": "0", + //"output_cache_hits": "0", + //"output_cache_misses": "18506478", + //"uri_cache_count": "2", + //"uri_cache_hits": "18506452", + //"uri_cache_misses": "26", + //"total_uris_cached": "13" + //} - Config = perfmon.Config{ - IgnoreNECounters: true, - CounterConfig: GetPerfCounters("webserver"), } ) + +func InitConfig(metricset string) perfmon.Config { + return perfmon.Config{ + IgnoreNECounters: true, + CounterConfig: GetPerfCounters(metricset), +} +} + + + func GetPerfCounters(metricset string) []perfmon.CounterConfig { - counters = []perfmon.CounterConfig{} + counters := []perfmon.CounterConfig{} switch metricset { - case "webserver": - for k, v := range webservice_counters { + case "website": + for k, v := range website_counters { counter := perfmon.CounterConfig{ InstanceLabel: "website.name", MeasurementLabel: k, Query: v, Format: "float", } - counters = append(counters, k, v) + counters = append(counters, counter) } - case "app_pool": - for k, v := range webservice_counters { + case "webserver": + for k, v := range webserver_counters { counter := perfmon.CounterConfig{ InstanceLabel: "website.name", MeasurementLabel: k, Query: v, Format: "float", } - counters = append(counters, k, v) + counters = append(counters, counter) } } return counters; } +func MapCounter(v interface{}) (common.MapStr, error){ + switch m := v.(type) { + case common.MapStr: + return m, nil + case map[string]interface{}: + return common.MapStr(m), nil + default: + return nil, errors.New("expected map but type is") + } +} + + + diff --git a/metricbeat/module/iis/system/_meta/docs.asciidoc b/metricbeat/module/iis/system/_meta/docs.asciidoc deleted file mode 100644 index db94dc7c8e80..000000000000 --- a/metricbeat/module/iis/system/_meta/docs.asciidoc +++ /dev/null @@ -1 +0,0 @@ -This is the system metricset of the module iis. diff --git a/metricbeat/module/iis/system/data.go b/metricbeat/module/iis/system/data.go deleted file mode 100644 index 67787e1915eb..000000000000 --- a/metricbeat/module/iis/system/data.go +++ /dev/null @@ -1,17 +0,0 @@ -package system - -import ( - "github.com/elastic/beats/metricbeat/mb" -) - - - -func eventsMapping(r mb.ReporterV2, events []mb.Event) { - - //remapping process details to match the naming format - for _, event := range events { - - - r.Event(event) - } -} diff --git a/metricbeat/module/iis/webserver/_meta/data.json b/metricbeat/module/iis/webserver/_meta/data.json new file mode 100644 index 000000000000..49f09a5610e1 --- /dev/null +++ b/metricbeat/module/iis/webserver/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"iis", + "name":"webserver", + "rtt":44269 + }, + "iis":{ + "webserver":{ + "example": "webserver" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/iis/webserver/_meta/docs.asciidoc b/metricbeat/module/iis/webserver/_meta/docs.asciidoc new file mode 100644 index 000000000000..8c0c2ea43b10 --- /dev/null +++ b/metricbeat/module/iis/webserver/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the webserver metricset of the module iis. diff --git a/metricbeat/module/iis/webserver/_meta/fields.yml b/metricbeat/module/iis/webserver/_meta/fields.yml new file mode 100644 index 000000000000..931219e6f14d --- /dev/null +++ b/metricbeat/module/iis/webserver/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: webserver + type: group + description: > + webserver + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/iis/webserver/data.go b/metricbeat/module/iis/webserver/data.go new file mode 100644 index 000000000000..eef9d976fd62 --- /dev/null +++ b/metricbeat/module/iis/webserver/data.go @@ -0,0 +1,44 @@ +package webserver + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/iis" +) + + +func eventsMapping(r mb.ReporterV2, events []mb.Event) { + + //remapping process details to match the naming format + groupedEvents := make(map[interface{}][]mb.Event) + for _, event := range events { + website, err := event.MetricSetFields.GetValue("website.name") + if err != nil { + r.Error(err) + return + } + groupedEvents[website] = append(groupedEvents[website], event) + } + for _, grouped := range groupedEvents { + counters := common.MapStr{} + for _, event := range grouped { + counterResults, err := event.MetricSetFields.GetValue("webserver") + if err != nil { + r.Error(err) + return + } + mapCounterResults, err := iis.MapCounter(counterResults) + if err != nil { + r.Error(err) + return + } + common.MergeFields(counters, mapCounterResults, true) + } + groupedEvent := mb.Event{MetricSetFields: counters} + r.Event(groupedEvent) + } + } + + + + diff --git a/metricbeat/module/iis/webserver/webserver.go b/metricbeat/module/iis/webserver/webserver.go new file mode 100644 index 000000000000..0a6e16a1f92a --- /dev/null +++ b/metricbeat/module/iis/webserver/webserver.go @@ -0,0 +1,55 @@ +package webserver + +import ( + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/iis" + "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "github.com/pkg/errors" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("iis", "webserver", New, mb.DefaultMetricSet()) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + reader *perfmon.PerfmonReader +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The iis webserver metricset is experimental.") + config := iis.InitConfig("webserver") + + reader, err := perfmon.NewPerfmonReader(config) + if err != nil { + return nil, errors.Wrap(err, "initialization of reader failed") + } + + return &MetricSet{ + BaseMetricSet: base, + reader : reader, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + events, err := m.reader.Read() + if err != nil { + err = errors.Wrap(err, "failed reading counters") + report.Error(err) + } + eventsMapping(report, events) +} diff --git a/metricbeat/module/iis/system/_meta/data.json b/metricbeat/module/iis/website/_meta/data.json similarity index 77% rename from metricbeat/module/iis/system/_meta/data.json rename to metricbeat/module/iis/website/_meta/data.json index b44caf91042a..32874cf585bd 100644 --- a/metricbeat/module/iis/system/_meta/data.json +++ b/metricbeat/module/iis/website/_meta/data.json @@ -7,12 +7,12 @@ "metricset":{ "host":"localhost", "module":"iis", - "name":"system", + "name":"website", "rtt":44269 }, "iis":{ - "system":{ - "example": "system" + "website":{ + "example": "website" } }, "type":"metricsets" diff --git a/metricbeat/module/iis/website/_meta/docs.asciidoc b/metricbeat/module/iis/website/_meta/docs.asciidoc new file mode 100644 index 000000000000..b2ea5dab2da3 --- /dev/null +++ b/metricbeat/module/iis/website/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the website metricset of the module iis. diff --git a/metricbeat/module/iis/system/_meta/fields.yml b/metricbeat/module/iis/website/_meta/fields.yml similarity index 81% rename from metricbeat/module/iis/system/_meta/fields.yml rename to metricbeat/module/iis/website/_meta/fields.yml index ecbd7ffc7869..7c84d6909d61 100644 --- a/metricbeat/module/iis/system/_meta/fields.yml +++ b/metricbeat/module/iis/website/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: system +- name: website type: group description: > - system + website fields: - name: example type: keyword diff --git a/metricbeat/module/iis/website/data.go b/metricbeat/module/iis/website/data.go new file mode 100644 index 000000000000..60a8c77603a4 --- /dev/null +++ b/metricbeat/module/iis/website/data.go @@ -0,0 +1,40 @@ +package website + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/iis" +) + + +func eventsMapping(r mb.ReporterV2, events []mb.Event) { + + //remapping process details to match the naming format + groupedEvents := make(map[interface{}][]mb.Event) + for _, event := range events { + website, err := event.MetricSetFields.GetValue("website.name") + if err != nil { + r.Error(err) + return + } + groupedEvents[website] = append(groupedEvents[website], event) + } + for _, grouped := range groupedEvents { + counters := common.MapStr{} + for _, event := range grouped { + counterResults, err := event.MetricSetFields.GetValue("website") + if err != nil { + r.Error(err) + return + } + mapCounterResults, err := iis.MapCounter(counterResults) + if err != nil { + r.Error(err) + return + } + common.MergeFields(counters, mapCounterResults, true) + } + groupedEvent := mb.Event{MetricSetFields: counters} + r.Event(groupedEvent) + } +} diff --git a/metricbeat/module/iis/system/system.go b/metricbeat/module/iis/website/website.go similarity index 86% rename from metricbeat/module/iis/system/system.go rename to metricbeat/module/iis/website/website.go index b0175e983ed1..a7b3284210de 100644 --- a/metricbeat/module/iis/system/system.go +++ b/metricbeat/module/iis/website/website.go @@ -1,4 +1,4 @@ -package system +package website import ( "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -13,7 +13,7 @@ import ( // the MetricSet for each host defined in the module's configuration. After the // MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("iis", "system", New, mb.DefaultMetricSet()) + mb.Registry.MustAddMetricSet("iis", "website", New) } // MetricSet holds any configuration or state information. It must implement @@ -28,12 +28,9 @@ type MetricSet struct { // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Experimental("The iis system metricset is experimental.") + cfgwarn.Experimental("The iis website metricset is experimental.") - config := iis.Config - //if err := base.Module().UnpackConfig(&config); err != nil { - // return nil, err - //} + config := iis.InitConfig("website") reader, err := perfmon.NewPerfmonReader(config) if err != nil { diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 9487a58b7cff..5e5b9b4ca24f 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -1,7 +1,7 @@ - module: system metricsets: - cpu # CPU usage - - load # CPU load averages + #- load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml.disabled similarity index 75% rename from metricbeat/modules.d/iis.yml rename to metricbeat/modules.d/iis.yml.disabled index 1dc41125c69e..1dd49c94be14 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml.disabled @@ -2,8 +2,9 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-iis.html - module: iis - metricsets: ["system"] + metricsets: + - webserver + - website enabled: true period: 10s - hosts: ["localhost"] diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 1b1413b31c35..b4d38cc520bc 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -5,7 +5,7 @@ period: 10s metricsets: - cpu - #- load + - load - memory - network - process From b1fba8f1ede755792a802bf02340170af5e43f7a Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 11 Dec 2019 14:18:34 +0100 Subject: [PATCH 03/36] iis changes --- metricbeat/module/iis/helper.go | 133 ----------- metricbeat/module/iis/performanceCounters.go | 90 ++++++++ metricbeat/module/iis/reader.go | 209 ++++++++++++++++++ metricbeat/module/iis/webserver/webserver.go | 9 +- metricbeat/module/iis/website/website.go | 9 +- .../perfmon/pdh_integration_windows_test.go | 6 +- .../windows/perfmon/pdh_query_windows.go | 8 +- .../windows/perfmon/pdh_query_windows_test.go | 4 +- metricbeat/module/windows/perfmon/reader.go | 4 +- .../modules.d/{iis.yml.disabled => iis.yml} | 0 .../{system.yml => system.yml.disabled} | 0 11 files changed, 317 insertions(+), 155 deletions(-) delete mode 100644 metricbeat/module/iis/helper.go create mode 100644 metricbeat/module/iis/performanceCounters.go create mode 100644 metricbeat/module/iis/reader.go rename metricbeat/modules.d/{iis.yml.disabled => iis.yml} (100%) rename metricbeat/modules.d/{system.yml => system.yml.disabled} (100%) diff --git a/metricbeat/module/iis/helper.go b/metricbeat/module/iis/helper.go deleted file mode 100644 index c1cbf065ca5c..000000000000 --- a/metricbeat/module/iis/helper.go +++ /dev/null @@ -1,133 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package iis - -import ( - "github.com/elastic/beats/metricbeat/module/windows/perfmon" - "github.com/elastic/beats/libbeat/common" - "errors" -) - -var ( - - website_counters = map[string]string { - "website.bytes_sent_per_sec":"\\Web Service(*)\\Bytes Sent/sec", - "website.total_bytes_sent_per_sec":"\\Web Service(*)\\Total Bytes Sent", - "website.bytes_recv_per_sec":"\\Web Service(*)\\Bytes Received/sec", - "website.total_bytes_recv_per_sec":"\\Web Service(*)\\Total Bytes Received", - - //\Web Service(*)\Total Files Sent - //\Web Service(*)\Files Sent/sec - //\Web Service(*)\Total Files Received - //\Web Service(*)\Files Received/sec - //\Web Service(*)\Current Connections - //\Web Service(*)\Maximum Connections - //\Web Service(*)\Total Connection Attempts (all instances) - //\Web Service(*)\Total Get Requests - //\Web Service(*)\Get Requests/sec - //\Web Service(*)\Total Post Requests - //\Web Service(*)\Post Requests/sec -} - webserver_counters = map[string]string { - "webserver.total_bytes_sent_per_sec":"\\Web Service(_Total)\\Total Bytes Sent", - "webserver.total_bytes_recv_per_sec":"\\Web Service(_Total)\\Total Bytes Received", - //\Web Service(*)\Total Files Sent - //\Web Service(*)\Files Sent/sec - //\Web Service(*)\Total Files Received - //\Web Service(*)\Files Received/sec - //\Web Service(*)\Current Connections - //\Web Service(*)\Maximum Connections - //\Web Service(*)\Total Connection Attempts (all instances) - //\Web Service(*)\Total Get Requests - //\Web Service(*)\Get Requests/sec - //\Web Service(*)\Total Post Requests - //\Web Service(*)\Post Requests/sec - - //cache - //"cache": { - //"file_cache_count": "2", - //"file_cache_memory_usage": "699", - //"file_cache_hits": "18506471", - //"file_cache_misses": "46266060", - //"total_files_cached": "10", - //"output_cache_count": "0", - //"output_cache_memory_usage": "0", - //"output_cache_hits": "0", - //"output_cache_misses": "18506478", - //"uri_cache_count": "2", - //"uri_cache_hits": "18506452", - //"uri_cache_misses": "26", - //"total_uris_cached": "13" - //} - - - } -) - -func InitConfig(metricset string) perfmon.Config { - return perfmon.Config{ - IgnoreNECounters: true, - CounterConfig: GetPerfCounters(metricset), -} -} - - - -func GetPerfCounters(metricset string) []perfmon.CounterConfig { - counters := []perfmon.CounterConfig{} - switch metricset { - case "website": - for k, v := range website_counters { - counter := perfmon.CounterConfig{ - InstanceLabel: "website.name", - MeasurementLabel: k, - Query: v, - Format: "float", - } - counters = append(counters, counter) - } - case "webserver": - for k, v := range webserver_counters { - counter := perfmon.CounterConfig{ - InstanceLabel: "website.name", - MeasurementLabel: k, - Query: v, - Format: "float", - } - counters = append(counters, counter) - } - - } - return counters; -} - -func MapCounter(v interface{}) (common.MapStr, error){ - switch m := v.(type) { - case common.MapStr: - return m, nil - case map[string]interface{}: - return common.MapStr(m), nil - default: - return nil, errors.New("expected map but type is") - } -} - - - diff --git a/metricbeat/module/iis/performanceCounters.go b/metricbeat/module/iis/performanceCounters.go new file mode 100644 index 000000000000..0b3c5275e678 --- /dev/null +++ b/metricbeat/module/iis/performanceCounters.go @@ -0,0 +1,90 @@ +package iis + +var ( + website_counters = map[string]string{ + "website.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", + "website.total_bytes_sent_per_sec": "\\Web Service(*)\\Total Bytes Sent", + "website.bytes_recv_per_sec": "\\Web Service(*)\\Bytes Received/sec", + "website.total_bytes_recv_per_sec": "\\Web Service(*)\\Total Bytes Received", + + //\Web Service(*)\Total Files Sent + //\Web Service(*)\Files Sent/sec + //\Web Service(*)\Total Files Received + //\Web Service(*)\Files Received/sec + //\Web Service(*)\Current Connections + //\Web Service(*)\Maximum Connections + //\Web Service(*)\Total Connection Attempts (all instances) + //\Web Service(*)\Total Get Requests + //\Web Service(*)\Get Requests/sec + //\Web Service(*)\Total Post Requests + //\Web Service(*)\Post Requests/sec + } + webserver_counters = map[string]string{ + "webserver.total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", + "webserver.total_bytes_recv_per_sec": "\\Web Service(_Total)\\Total Bytes Received", + //\Web Service(*)\Total Files Sent + //\Web Service(*)\Files Sent/sec + //\Web Service(*)\Total Files Received + //\Web Service(*)\Files Received/sec + //\Web Service(*)\Current Connections + //\Web Service(*)\Maximum Connections + //\Web Service(*)\Total Connection Attempts (all instances) + //\Web Service(*)\Total Get Requests + //\Web Service(*)\Get Requests/sec + //\Web Service(*)\Total Post Requests + //\Web Service(*)\Post Requests/sec + + //cache + //"cache": { + //"file_cache_count": "2", + //"file_cache_memory_usage": "699", + //"file_cache_hits": "18506471", + //"file_cache_misses": "46266060", + //"total_files_cached": "10", + //"output_cache_count": "0", + //"output_cache_memory_usage": "0", + //"output_cache_hits": "0", + //"output_cache_misses": "18506478", + //"uri_cache_count": "2", + //"uri_cache_hits": "18506452", + //"uri_cache_misses": "26", + //"total_uris_cached": "13" + //} + + } +) + +type PerformanceCounter struct { + InstanceLabel string + MeasurementLabel string + Path string + Format string +} + +func GetPerfCounters(metricset string) []PerformanceCounter { + var counters []PerformanceCounter + switch metricset { + case "website": + for k, v := range website_counters { + counter := PerformanceCounter{ + InstanceLabel: "website.name", + MeasurementLabel: k, + Path: v, + Format: "float", + } + counters = append(counters, counter) + } + case "webserver": + for k, v := range webserver_counters { + counter := PerformanceCounter{ + InstanceLabel: "webserver.name", + MeasurementLabel: k, + Path: v, + Format: "float", + } + counters = append(counters, counter) + } + + } + return counters +} diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/reader.go new file mode 100644 index 000000000000..f9a7192742e0 --- /dev/null +++ b/metricbeat/module/iis/reader.go @@ -0,0 +1,209 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package iis + +import ( + "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" +) + +var ( + processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) +) + +// Reader will contain the config options +type Reader struct { + query perfmon.Query // PDH Query + instanceLabel map[string]string // Mapping of counter path to key used for the label (e.g. processor.name) + measurement map[string]string // Mapping of counter path to key used for the value (e.g. processor.cpu_time). + executed bool // Indicates if the query has been executed. + log *logp.Logger // + counters []PerformanceCounter +} + +// NewReader creates a new instance of Reader. +func NewReader(counters []PerformanceCounter) (*Reader, error) { + var query perfmon.Query + if err := query.Open(); err != nil { + return nil, err + } + r := &Reader{ + query: query, + instanceLabel: map[string]string{}, + measurement: map[string]string{}, + log: logp.NewLogger("iis"), + counters: counters, + } + for _, counter := range counters { + childQueries, err := query.GetCounterPaths(counter.Path) + if err != nil { + if err == perfmon.PDH_CSTATUS_NO_COUNTER || err == perfmon.PDH_CSTATUS_NO_COUNTERNAME || err == perfmon.PDH_CSTATUS_NO_INSTANCE || err == perfmon.PDH_CSTATUS_NO_OBJECT { + r.log.Infow("Ignoring non existent counter", "error", err, + logp.Namespace("iis"), "query", counter.Path) + continue + } else { + query.Close() + return nil, errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Path) + } + } + // check if the pdhexpandcounterpath/pdhexpandwildcardpath functions have expanded the counter successfully. + if len(childQueries) == 0 || (len(childQueries) == 1 && strings.Contains(childQueries[0], "*")) { + // covering cases when PdhExpandWildCardPathW returns no counter paths or is unable to expand and the ignore_non_existent_counters flag is set + r.log.Infow("Ignoring non existent counter", "initial query", counter.Path, + logp.Namespace("perfmon"), "expanded query", childQueries) + continue + } + for _, v := range childQueries { + if err := query.AddCounter(v, counter.InstanceLabel, counter.Format, len(childQueries) > 1); err != nil { + return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Path) + } + r.instanceLabel[v] = counter.InstanceLabel + r.measurement[v] = counter.MeasurementLabel + } + } + + return r, nil +} + +// RefreshCounterPaths will recheck for any new instances and add them to the counter list +func (r *Reader) RefreshCounterPaths() error { + var newCounters []string + for _, counter := range r.counters { + childQueries, err := r.query.GetCounterPaths(counter.Path) + if err != nil { + if err == perfmon.PDH_CSTATUS_NO_COUNTER || err == perfmon.PDH_CSTATUS_NO_COUNTERNAME || err == perfmon.PDH_CSTATUS_NO_INSTANCE || err == perfmon.PDH_CSTATUS_NO_OBJECT { + r.log.Infow("Ignoring non existent counter", "error", err, + logp.Namespace("iis"), "query", counter.Path) + continue + } else { + return errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Path) + } + } + newCounters = append(newCounters, childQueries...) + // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list + if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { + for _, v := range childQueries { + if err := r.query.AddCounter(v, counter.InstanceLabel, counter.Format, len(childQueries) > 1); err != nil { + return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Path) + } + r.instanceLabel[v] = counter.InstanceLabel + r.measurement[v] = counter.MeasurementLabel + } + } + } + err := r.query.RemoveUnusedCounters(newCounters) + if err != nil { + return errors.Wrap(err, "failed removing unused counter values") + } + + return nil +} + +// Read executes a query and returns those values in an event. +func (r *Reader) Read() ([]mb.Event, error) { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if err := r.query.CollectData(); err != nil { + return nil, errors.Wrap(err, "failed querying counter values") + } + + // Get the values. + values, err := r.query.GetFormattedCounterValues() + if err != nil { + return nil, errors.Wrap(err, "failed formatting counter values") + } + + eventMap := make(map[string]*mb.Event) + + for counterPath, values := range values { + for ind, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !r.executed { + r.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + + var eventKey string + if false && val.Err == nil { + // Send measurements with the same instance label as part of the same event + eventKey = val.Instance + } else { + // Send every measurement as an individual event + // If a counter contains an error, it will always be sent as an individual event + eventKey = counterPath + strconv.Itoa(ind) + } + + // Create a new event if the key doesn't exist in the map + if _, ok := eventMap[eventKey]; !ok { + eventMap[eventKey] = &mb.Event{ + MetricSetFields: common.MapStr{}, + Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), + } + if val.Instance != "" { + //will ignore instance counter + if ok, match := matchesParentProcess(val.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], match) + } else { + eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], val.Instance) + } + } + } + event := eventMap[eventKey] + if val.Measurement != nil { + event.MetricSetFields.Put(r.measurement[counterPath], val.Measurement) + } else { + event.MetricSetFields.Put(r.measurement[counterPath], 0) + } + } + } + + // Write the values into the map. + events := make([]mb.Event, 0, len(eventMap)) + for _, val := range eventMap { + events = append(events, *val) + } + + r.executed = true + return events, nil +} + +// Close will close the PDH query for now. +func (r *Reader) Close() error { + return r.query.Close() +} + +// matchParentProcess will try to get the parent process name +func matchesParentProcess(instanceName string) (bool, string) { + matches := processRegexp.FindStringSubmatch(instanceName) + if len(matches) == 2 { + return true, matches[1] + } + return false, instanceName +} diff --git a/metricbeat/module/iis/webserver/webserver.go b/metricbeat/module/iis/webserver/webserver.go index 0a6e16a1f92a..d973860a6d34 100644 --- a/metricbeat/module/iis/webserver/webserver.go +++ b/metricbeat/module/iis/webserver/webserver.go @@ -4,7 +4,6 @@ import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/iis" - "github.com/elastic/beats/metricbeat/module/windows/perfmon" "github.com/pkg/errors" ) @@ -22,23 +21,23 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - reader *perfmon.PerfmonReader + reader *iis.Reader } // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The iis webserver metricset is experimental.") - config := iis.InitConfig("webserver") + counters := iis.GetPerfCounters("webserver") - reader, err := perfmon.NewPerfmonReader(config) + reader, err := iis.NewReader(counters) if err != nil { return nil, errors.Wrap(err, "initialization of reader failed") } return &MetricSet{ BaseMetricSet: base, - reader : reader, + reader: reader, }, nil } diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index a7b3284210de..761af314fc35 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -4,7 +4,6 @@ import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/iis" - "github.com/elastic/beats/metricbeat/module/windows/perfmon" "github.com/pkg/errors" ) @@ -22,7 +21,7 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - reader *perfmon.PerfmonReader + reader *iis.Reader } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -30,16 +29,14 @@ type MetricSet struct { func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The iis website metricset is experimental.") - config := iis.InitConfig("website") - - reader, err := perfmon.NewPerfmonReader(config) + reader, err := iis.NewReader(iis.GetPerfCounters("website")) if err != nil { return nil, errors.Wrap(err, "initialization of reader failed") } return &MetricSet{ BaseMetricSet: base, - reader : reader, + reader: reader, }, nil } diff --git a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go index 1fe12ab20d1f..92bc5ca712cf 100644 --- a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go +++ b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go @@ -117,7 +117,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - err = q.AddCounter(path[0], counter, false) + err = q.AddCounter(path[0], counter.InstanceName, counter.Format, false) if err != nil { t.Fatal(err) } @@ -243,7 +243,7 @@ func TestLongOutputFormat(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter, false) + err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) if err != nil && err != PDH_NO_MORE_DATA { t.Fatal(err) } @@ -283,7 +283,7 @@ func TestFloatOutputFormat(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter, false) + err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) if err != nil && err != PDH_NO_MORE_DATA { t.Fatal(err) } diff --git a/metricbeat/module/windows/perfmon/pdh_query_windows.go b/metricbeat/module/windows/perfmon/pdh_query_windows.go index eef0e727b16d..c9308628dcd1 100644 --- a/metricbeat/module/windows/perfmon/pdh_query_windows.go +++ b/metricbeat/module/windows/perfmon/pdh_query_windows.go @@ -72,20 +72,20 @@ func (q *Query) AddEnglishCounter(counterPath string) (PdhCounterHandle, error) } // AddCounter adds the specified counter to the query. -func (q *Query) AddCounter(counterPath string, counter CounterConfig, wildcard bool) error { +func (q *Query) AddCounter(counterPath string, counterInstance string, counterFormat string, wildcard bool) error { if _, found := q.counters[counterPath]; found { return nil } var err error var instanceName string // Extract the instance name from the counterPath. - if counter.InstanceName == "" || wildcard { + if counterInstance == "" || wildcard { instanceName, err = matchInstanceName(counterPath) if err != nil { return err } } else { - instanceName = counter.InstanceName + instanceName = counterInstance } h, err := PdhAddCounter(q.handle, counterPath, 0) if err != nil { @@ -95,7 +95,7 @@ func (q *Query) AddCounter(counterPath string, counter CounterConfig, wildcard b q.counters[counterPath] = &Counter{ handle: h, instanceName: instanceName, - format: getPDHFormat(counter.Format), + format: getPDHFormat(counterFormat), } return nil } diff --git a/metricbeat/module/windows/perfmon/pdh_query_windows_test.go b/metricbeat/module/windows/perfmon/pdh_query_windows_test.go index 4b5c87104992..c81ab210cd07 100644 --- a/metricbeat/module/windows/perfmon/pdh_query_windows_test.go +++ b/metricbeat/module/windows/perfmon/pdh_query_windows_test.go @@ -38,7 +38,7 @@ func TestAddCounterInvalidArgWhenQueryClosed(t *testing.T) { // if windows os language is ENG then err will be nil, else the GetCounterPaths will execute the AddCounter if assert.NoError(t, err) { counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"} - err = q.AddCounter(queryPath[0], counter, false) + err = q.AddCounter(queryPath[0], counter.InstanceName, counter.Format, false) assert.Error(t, err, PDH_INVALID_HANDLE) } else { assert.Error(t, err, PDH_INVALID_ARGUMENT) @@ -74,7 +74,7 @@ func TestSuccessfulQuery(t *testing.T) { if err != nil { t.Fatal(err) } - err = q.AddCounter(queryPath[0], counter, false) + err = q.AddCounter(queryPath[0], counter.InstanceName, counter.Format, false) if err != nil { t.Fatal(err) } diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index b7837862ffca..436acdd5dba3 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -85,7 +85,7 @@ func NewReader(config Config) (*Reader, error) { return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.Query) } for _, v := range childQueries { - if err := query.AddCounter(v, counter, len(childQueries) > 1); err != nil { + if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) } r.instanceLabel[v] = counter.InstanceLabel @@ -118,7 +118,7 @@ func (r *Reader) RefreshCounterPaths() error { // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { - if err := r.query.AddCounter(v, counter, len(childQueries) > 1); err != nil { + if err := r.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } r.instanceLabel[v] = counter.InstanceLabel diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml similarity index 100% rename from metricbeat/modules.d/iis.yml.disabled rename to metricbeat/modules.d/iis.yml diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml.disabled similarity index 100% rename from metricbeat/modules.d/system.yml rename to metricbeat/modules.d/system.yml.disabled From 7a8ff78bc6863eecdde20ba7f12e1be6bbf1e92b Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 11 Dec 2019 17:18:19 +0100 Subject: [PATCH 04/36] add iis module --- metricbeat/docs/fields.asciidoc | 1 - metricbeat/docs/modules_list.asciidoc | 4 ++ metricbeat/include/list_common.go | 3 + metricbeat/metricbeat.reference.yml | 11 +++- metricbeat/module/iis/_meta/fields.yml | 4 +- metricbeat/module/iis/fields.go | 2 +- metricbeat/module/iis/performanceCounters.go | 16 ++--- metricbeat/module/iis/reader.go | 44 ++++++------- .../module/iis/webserver/_meta/fields.yml | 2 + metricbeat/module/iis/webserver/data.go | 44 ------------- metricbeat/module/iis/webserver/webserver.go | 63 ++++++++++++++++--- .../module/iis/website/_meta/fields.yml | 1 + metricbeat/module/iis/website/data.go | 40 ------------ metricbeat/module/iis/website/website.go | 63 ++++++++++++++++--- 14 files changed, 165 insertions(+), 133 deletions(-) delete mode 100644 metricbeat/module/iis/webserver/data.go delete mode 100644 metricbeat/module/iis/website/data.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 73d7f506249b..3e14338bf011 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -15267,7 +15267,6 @@ server [[exported-fields-iis]] == iis fields -experimental[] iis module diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index c086069cb45a..39a24ac3836d 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -96,6 +96,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.2+| .2+| |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | @@ -242,6 +245,7 @@ include::modules/golang.asciidoc[] include::modules/graphite.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/http.asciidoc[] +include::modules/iis.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 26db25d57441..0ffeff23e859 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -76,6 +76,9 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http" _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" + _ "github.com/elastic/beats/metricbeat/module/iis" + _ "github.com/elastic/beats/metricbeat/module/iis/webserver" + _ "github.com/elastic/beats/metricbeat/module/iis/website" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9241da7a0661..c8f76abe443d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -60,7 +60,7 @@ metricbeat.modules: - module: system metricsets: - cpu # CPU usage - - load # CPU load averages + #- load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics @@ -342,6 +342,15 @@ metricbeat.modules: # fields: # added to the the response in root. overwrites existing fields # key: "value" +#--------------------------------- Iis Module --------------------------------- +- module: iis + metricsets: + - webserver + - website + enabled: true + period: 10s + + #------------------------------- Jolokia Module ------------------------------- - module: jolokia #metricsets: ["jmx"] diff --git a/metricbeat/module/iis/_meta/fields.yml b/metricbeat/module/iis/_meta/fields.yml index 382fd01c9fb3..9a7b776c59f6 100644 --- a/metricbeat/module/iis/_meta/fields.yml +++ b/metricbeat/module/iis/_meta/fields.yml @@ -1,9 +1,9 @@ - key: iis title: "iis" description: > - experimental[] - iis module + release: beta + fields: - name: iis type: group diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index 3bcad1e3f5cd..7fd087573eb6 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -32,5 +32,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/iis. func AssetIis() string { - return "eJzEj7FugzAYhHc/xYmdF/DQrU9RdaD1tfqFjS3bFHj7CpxEgJwsGXLjnfR9uhY9Fw2RpIAs2VKjEUmNAgzTd5SQxQ8abwoAOAdGcRxyZz8+1daJJDhvRksF/AitSXobWgyd4xW+Ji+BGr/Rj+HSVBxHyB408Ssx/jHelhryLrakBjkL91LOnQvbt32Ktucy+WhO2wP5mvcCLNLqR8l89uER8cJ//wEAAP//Ae+cTA==" + return "eJzEkDFuhTAQRHufYkTPBVyky0FMPIlWGGzZJoTbR+AQGeSk+cWfcnf0dnZ6jNw0RJICsmRHjU4kdQqwTG9RQhY/a7woALsPk7eLowIiHU2ixsBslALehc4mfTh7zGbiSd6Vt0CNj+iX8DNpHLhCatDKITF+Mv5uWsg/sUUtyPWNc1oZ7onqVPwyUzjaqFVyjdxWH+1t90+6Xa8FWI42S5DMRyu4ItoFPPXv7wAAAP//UtuoaQ==" } diff --git a/metricbeat/module/iis/performanceCounters.go b/metricbeat/module/iis/performanceCounters.go index 0b3c5275e678..fe71dcf56801 100644 --- a/metricbeat/module/iis/performanceCounters.go +++ b/metricbeat/module/iis/performanceCounters.go @@ -2,10 +2,10 @@ package iis var ( website_counters = map[string]string{ - "website.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", - "website.total_bytes_sent_per_sec": "\\Web Service(*)\\Total Bytes Sent", - "website.bytes_recv_per_sec": "\\Web Service(*)\\Bytes Received/sec", - "website.total_bytes_recv_per_sec": "\\Web Service(*)\\Total Bytes Received", + "bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", + "total_bytes_sent_per_sec": "\\Web Service(*)\\Total Bytes Sent", + "bytes_recv_per_sec": "\\Web Service(*)\\Bytes Received/sec", + "total_bytes_recv_per_sec": "\\Web Service(*)\\Total Bytes Received", //\Web Service(*)\Total Files Sent //\Web Service(*)\Files Sent/sec @@ -20,8 +20,8 @@ var ( //\Web Service(*)\Post Requests/sec } webserver_counters = map[string]string{ - "webserver.total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", - "webserver.total_bytes_recv_per_sec": "\\Web Service(_Total)\\Total Bytes Received", + "total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", + "total_bytes_recv_per_sec": "\\Web Service(_Total)\\Total Bytes Received", //\Web Service(*)\Total Files Sent //\Web Service(*)\Files Sent/sec //\Web Service(*)\Total Files Received @@ -67,7 +67,7 @@ func GetPerfCounters(metricset string) []PerformanceCounter { case "website": for k, v := range website_counters { counter := PerformanceCounter{ - InstanceLabel: "website.name", + InstanceLabel: "name", MeasurementLabel: k, Path: v, Format: "float", @@ -77,7 +77,7 @@ func GetPerfCounters(metricset string) []PerformanceCounter { case "webserver": for k, v := range webserver_counters { counter := PerformanceCounter{ - InstanceLabel: "webserver.name", + InstanceLabel: "", MeasurementLabel: k, Path: v, Format: "float", diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/reader.go index f9a7192742e0..0e597c313f02 100644 --- a/metricbeat/module/iis/reader.go +++ b/metricbeat/module/iis/reader.go @@ -22,7 +22,6 @@ package iis import ( "github.com/elastic/beats/metricbeat/module/windows/perfmon" "regexp" - "strconv" "strings" "github.com/pkg/errors" @@ -41,7 +40,7 @@ type Reader struct { query perfmon.Query // PDH Query instanceLabel map[string]string // Mapping of counter path to key used for the label (e.g. processor.name) measurement map[string]string // Mapping of counter path to key used for the value (e.g. processor.cpu_time). - executed bool // Indicates if the query has been executed. + Executed bool // Indicates if the query has been executed. log *logp.Logger // counters []PerformanceCounter } @@ -125,7 +124,7 @@ func (r *Reader) RefreshCounterPaths() error { } // Read executes a query and returns those values in an event. -func (r *Reader) Read() ([]mb.Event, error) { +func (r *Reader) Read(metricsetName string) ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if err := r.query.CollectData(); err != nil { @@ -140,46 +139,47 @@ func (r *Reader) Read() ([]mb.Event, error) { eventMap := make(map[string]*mb.Event) - for counterPath, values := range values { - for ind, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !r.executed { + for path, counterValues := range values { + for _, counterValue := range counterValues { + if counterValue.Err != nil && !r.Executed { r.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + "error", counterValue.Err, logp.Namespace("iis"), "query", path) + continue + } + if counterValue.Err != nil { + r.log.Errorw("Error while retrieving counter values", + "error", counterValue.Err, logp.Namespace("perfmon"), "query", path) continue } - var eventKey string - if false && val.Err == nil { + if metricsetName == "webserver" { // Send measurements with the same instance label as part of the same event - eventKey = val.Instance + eventKey = metricsetName } else { // Send every measurement as an individual event // If a counter contains an error, it will always be sent as an individual event - eventKey = counterPath + strconv.Itoa(ind) + eventKey = counterValue.Instance } // Create a new event if the key doesn't exist in the map if _, ok := eventMap[eventKey]; !ok { eventMap[eventKey] = &mb.Event{ MetricSetFields: common.MapStr{}, - Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), } - if val.Instance != "" { + if metricsetName != "webserver" { //will ignore instance counter - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], match) + if ok, match := matchesParentProcess(counterValue.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[path], match) } else { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], val.Instance) + eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[path], counterValue.Instance) } } } event := eventMap[eventKey] - if val.Measurement != nil { - event.MetricSetFields.Put(r.measurement[counterPath], val.Measurement) + if counterValue.Measurement != nil { + event.MetricSetFields.Put(r.measurement[path], counterValue.Measurement) } else { - event.MetricSetFields.Put(r.measurement[counterPath], 0) + event.MetricSetFields.Put(r.measurement[path], 0) } } } @@ -190,7 +190,7 @@ func (r *Reader) Read() ([]mb.Event, error) { events = append(events, *val) } - r.executed = true + r.Executed = true return events, nil } diff --git a/metricbeat/module/iis/webserver/_meta/fields.yml b/metricbeat/module/iis/webserver/_meta/fields.yml index 931219e6f14d..92d0501fb34d 100644 --- a/metricbeat/module/iis/webserver/_meta/fields.yml +++ b/metricbeat/module/iis/webserver/_meta/fields.yml @@ -2,6 +2,8 @@ type: group description: > webserver + release: beta + fields: - name: example type: keyword diff --git a/metricbeat/module/iis/webserver/data.go b/metricbeat/module/iis/webserver/data.go deleted file mode 100644 index eef9d976fd62..000000000000 --- a/metricbeat/module/iis/webserver/data.go +++ /dev/null @@ -1,44 +0,0 @@ -package webserver - -import ( - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/iis" -) - - -func eventsMapping(r mb.ReporterV2, events []mb.Event) { - - //remapping process details to match the naming format - groupedEvents := make(map[interface{}][]mb.Event) - for _, event := range events { - website, err := event.MetricSetFields.GetValue("website.name") - if err != nil { - r.Error(err) - return - } - groupedEvents[website] = append(groupedEvents[website], event) - } - for _, grouped := range groupedEvents { - counters := common.MapStr{} - for _, event := range grouped { - counterResults, err := event.MetricSetFields.GetValue("webserver") - if err != nil { - r.Error(err) - return - } - mapCounterResults, err := iis.MapCounter(counterResults) - if err != nil { - r.Error(err) - return - } - common.MergeFields(counters, mapCounterResults, true) - } - groupedEvent := mb.Event{MetricSetFields: counters} - r.Event(groupedEvent) - } - } - - - - diff --git a/metricbeat/module/iis/webserver/webserver.go b/metricbeat/module/iis/webserver/webserver.go index d973860a6d34..27a823b87399 100644 --- a/metricbeat/module/iis/webserver/webserver.go +++ b/metricbeat/module/iis/webserver/webserver.go @@ -1,3 +1,22 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + package webserver import ( @@ -7,12 +26,14 @@ import ( "github.com/pkg/errors" ) +const metricsetName = "webserver" + // init registers the MetricSet with the central registry as soon as the program // starts. The New function will be called later to instantiate an instance of // the MetricSet for each host defined in the module's configuration. After the // MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("iis", "webserver", New, mb.DefaultMetricSet()) + mb.Registry.MustAddMetricSet("iis", metricsetName, New, mb.DefaultMetricSet()) } // MetricSet holds any configuration or state information. It must implement @@ -28,7 +49,7 @@ type MetricSet struct { // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The iis webserver metricset is experimental.") - counters := iis.GetPerfCounters("webserver") + counters := iis.GetPerfCounters(metricsetName) reader, err := iis.NewReader(counters) if err != nil { @@ -44,11 +65,39 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(report mb.ReporterV2) { - events, err := m.reader.Read() +// Fetch fetches events and reports them upstream +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + + // refresh performance counter list + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + // A flag is set if the second call has been executed else refresh will fail (reader.executed) + if m.reader.Executed { + err := m.reader.RefreshCounterPaths() + if err != nil { + return errors.Wrap(err, "failed retrieving counters") + } + } + + events, err := m.reader.Read(metricsetName) + if err != nil { + return errors.Wrap(err, "failed reading counters") + } + + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } + } + return nil +} + +// Close will be called when metricbeat is stopped, should close the query. +func (m *MetricSet) Close() error { + err := m.reader.Close() if err != nil { - err = errors.Wrap(err, "failed reading counters") - report.Error(err) + return errors.Wrap(err, "failed to close pdh query") } - eventsMapping(report, events) + return nil } diff --git a/metricbeat/module/iis/website/_meta/fields.yml b/metricbeat/module/iis/website/_meta/fields.yml index 7c84d6909d61..c88aa44c536a 100644 --- a/metricbeat/module/iis/website/_meta/fields.yml +++ b/metricbeat/module/iis/website/_meta/fields.yml @@ -2,6 +2,7 @@ type: group description: > website + release: beta fields: - name: example type: keyword diff --git a/metricbeat/module/iis/website/data.go b/metricbeat/module/iis/website/data.go deleted file mode 100644 index 60a8c77603a4..000000000000 --- a/metricbeat/module/iis/website/data.go +++ /dev/null @@ -1,40 +0,0 @@ -package website - -import ( - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/iis" -) - - -func eventsMapping(r mb.ReporterV2, events []mb.Event) { - - //remapping process details to match the naming format - groupedEvents := make(map[interface{}][]mb.Event) - for _, event := range events { - website, err := event.MetricSetFields.GetValue("website.name") - if err != nil { - r.Error(err) - return - } - groupedEvents[website] = append(groupedEvents[website], event) - } - for _, grouped := range groupedEvents { - counters := common.MapStr{} - for _, event := range grouped { - counterResults, err := event.MetricSetFields.GetValue("website") - if err != nil { - r.Error(err) - return - } - mapCounterResults, err := iis.MapCounter(counterResults) - if err != nil { - r.Error(err) - return - } - common.MergeFields(counters, mapCounterResults, true) - } - groupedEvent := mb.Event{MetricSetFields: counters} - r.Event(groupedEvent) - } -} diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index 761af314fc35..f13ae1077ce8 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -1,3 +1,22 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + package website import ( @@ -7,12 +26,14 @@ import ( "github.com/pkg/errors" ) +const metricsetName= "website" + // init registers the MetricSet with the central registry as soon as the program // starts. The New function will be called later to instantiate an instance of // the MetricSet for each host defined in the module's configuration. After the // MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("iis", "website", New) + mb.Registry.MustAddMetricSet("iis", metricsetName, New) } // MetricSet holds any configuration or state information. It must implement @@ -29,7 +50,7 @@ type MetricSet struct { func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The iis website metricset is experimental.") - reader, err := iis.NewReader(iis.GetPerfCounters("website")) + reader, err := iis.NewReader(iis.GetPerfCounters(metricsetName)) if err != nil { return nil, errors.Wrap(err, "initialization of reader failed") } @@ -43,11 +64,39 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(report mb.ReporterV2) { - events, err := m.reader.Read() +// Fetch fetches events and reports them upstream +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + + // refresh performance counter list + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + // A flag is set if the second call has been executed else refresh will fail (reader.executed) + if m.reader.Executed { + err := m.reader.RefreshCounterPaths() + if err != nil { + return errors.Wrap(err, "failed retrieving counters") + } + } + events, err := m.reader.Read(metricsetName) + if err != nil { + return errors.Wrap(err, "failed reading counters") + } + + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } + } + return nil +} + +// Close will be called when metricbeat is stopped, should close the query. +func (m *MetricSet) Close() error { + err := m.reader.Close() if err != nil { - err = errors.Wrap(err, "failed reading counters") - report.Error(err) + return errors.Wrap(err, "failed to close pdh query") } - eventsMapping(report, events) + return nil } + From 6bd81ebc60e2d39890c3a9749015b8b5a2e13741 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Jan 2020 16:50:06 +0100 Subject: [PATCH 05/36] light --- metricbeat/docs/fields.asciidoc | 22 -- metricbeat/docs/modules/activemq.asciidoc | 86 +++++++ .../docs/modules/activemq/broker.asciidoc | 24 ++ .../docs/modules/activemq/queue.asciidoc | 24 ++ .../docs/modules/activemq/topic.asciidoc | 24 ++ metricbeat/docs/modules/aerospike.asciidoc | 48 ++++ .../docs/modules/aerospike/namespace.asciidoc | 22 ++ metricbeat/docs/modules/apache.asciidoc | 64 +++++ .../docs/modules/apache/status.asciidoc | 22 ++ metricbeat/docs/modules/appsearch.asciidoc | 41 +++ .../docs/modules/appsearch/stats.asciidoc | 24 ++ metricbeat/docs/modules/aws.asciidoc | 195 +++++++++++++++ metricbeat/docs/modules/aws/billing.asciidoc | 24 ++ .../docs/modules/aws/cloudwatch.asciidoc | 22 ++ metricbeat/docs/modules/aws/ebs.asciidoc | 22 ++ metricbeat/docs/modules/aws/ec2.asciidoc | 22 ++ metricbeat/docs/modules/aws/elb.asciidoc | 24 ++ metricbeat/docs/modules/aws/rds.asciidoc | 21 ++ .../modules/aws/s3_daily_storage.asciidoc | 21 ++ .../docs/modules/aws/s3_request.asciidoc | 21 ++ metricbeat/docs/modules/aws/sns.asciidoc | 24 ++ metricbeat/docs/modules/aws/sqs.asciidoc | 21 ++ metricbeat/docs/modules/aws/usage.asciidoc | 24 ++ metricbeat/docs/modules/azure.asciidoc | 151 +++++++++++ .../docs/modules/azure/compute_vm.asciidoc | 23 ++ .../azure/compute_vm_scaleset.asciidoc | 23 ++ .../docs/modules/azure/monitor.asciidoc | 24 ++ metricbeat/docs/modules/beat.asciidoc | 55 ++++ metricbeat/docs/modules/beat/state.asciidoc | 21 ++ metricbeat/docs/modules/beat/stats.asciidoc | 21 ++ metricbeat/docs/modules/ceph.asciidoc | 76 ++++++ .../docs/modules/ceph/cluster_disk.asciidoc | 22 ++ .../docs/modules/ceph/cluster_health.asciidoc | 22 ++ .../docs/modules/ceph/cluster_status.asciidoc | 22 ++ .../docs/modules/ceph/monitor_health.asciidoc | 22 ++ metricbeat/docs/modules/ceph/osd_df.asciidoc | 22 ++ .../docs/modules/ceph/osd_tree.asciidoc | 22 ++ .../docs/modules/ceph/pool_disk.asciidoc | 22 ++ metricbeat/docs/modules/cockroachdb.asciidoc | 59 +++++ .../docs/modules/cockroachdb/status.asciidoc | 24 ++ metricbeat/docs/modules/consul.asciidoc | 51 ++++ metricbeat/docs/modules/consul/agent.asciidoc | 24 ++ metricbeat/docs/modules/coredns.asciidoc | 51 ++++ .../docs/modules/coredns/stats.asciidoc | 21 ++ metricbeat/docs/modules/couchbase.asciidoc | 60 +++++ .../docs/modules/couchbase/bucket.asciidoc | 22 ++ .../docs/modules/couchbase/cluster.asciidoc | 22 ++ .../docs/modules/couchbase/node.asciidoc | 22 ++ metricbeat/docs/modules/couchdb.asciidoc | 49 ++++ .../docs/modules/couchdb/server.asciidoc | 22 ++ metricbeat/docs/modules/docker.asciidoc | 104 ++++++++ .../docs/modules/docker/container.asciidoc | 22 ++ metricbeat/docs/modules/docker/cpu.asciidoc | 22 ++ .../docs/modules/docker/diskio.asciidoc | 22 ++ metricbeat/docs/modules/docker/event.asciidoc | 22 ++ .../docs/modules/docker/healthcheck.asciidoc | 22 ++ metricbeat/docs/modules/docker/image.asciidoc | 21 ++ metricbeat/docs/modules/docker/info.asciidoc | 22 ++ .../docs/modules/docker/memory.asciidoc | 22 ++ .../docs/modules/docker/network.asciidoc | 22 ++ metricbeat/docs/modules/dropwizard.asciidoc | 45 ++++ .../modules/dropwizard/collector.asciidoc | 22 ++ .../docs/modules/elasticsearch.asciidoc | 108 ++++++++ .../docs/modules/elasticsearch/ccr.asciidoc | 21 ++ .../elasticsearch/cluster_stats.asciidoc | 21 ++ .../modules/elasticsearch/enrich.asciidoc | 21 ++ .../docs/modules/elasticsearch/index.asciidoc | 21 ++ .../elasticsearch/index_recovery.asciidoc | 21 ++ .../elasticsearch/index_summary.asciidoc | 21 ++ .../modules/elasticsearch/ml_job.asciidoc | 21 ++ .../docs/modules/elasticsearch/node.asciidoc | 22 ++ .../modules/elasticsearch/node_stats.asciidoc | 22 ++ .../elasticsearch/pending_tasks.asciidoc | 22 ++ .../docs/modules/elasticsearch/shard.asciidoc | 22 ++ metricbeat/docs/modules/envoyproxy.asciidoc | 41 +++ .../docs/modules/envoyproxy/server.asciidoc | 22 ++ metricbeat/docs/modules/etcd.asciidoc | 60 +++++ metricbeat/docs/modules/etcd/leader.asciidoc | 22 ++ metricbeat/docs/modules/etcd/metrics.asciidoc | 23 ++ metricbeat/docs/modules/etcd/self.asciidoc | 22 ++ metricbeat/docs/modules/etcd/store.asciidoc | 22 ++ metricbeat/docs/modules/golang.asciidoc | 47 ++++ .../docs/modules/golang/expvar.asciidoc | 21 ++ metricbeat/docs/modules/golang/heap.asciidoc | 21 ++ metricbeat/docs/modules/graphite.asciidoc | 54 ++++ .../docs/modules/graphite/server.asciidoc | 22 ++ metricbeat/docs/modules/haproxy.asciidoc | 77 ++++++ metricbeat/docs/modules/haproxy/info.asciidoc | 22 ++ metricbeat/docs/modules/haproxy/stat.asciidoc | 22 ++ metricbeat/docs/modules/http.asciidoc | 72 ++++++ metricbeat/docs/modules/http/json.asciidoc | 21 ++ metricbeat/docs/modules/http/server.asciidoc | 21 ++ metricbeat/docs/modules/iis.asciidoc | 43 ++++ .../docs/modules/iis/webserver.asciidoc | 24 ++ metricbeat/docs/modules/iis/website.asciidoc | 24 ++ metricbeat/docs/modules/jolokia.asciidoc | 74 ++++++ metricbeat/docs/modules/jolokia/jmx.asciidoc | 21 ++ metricbeat/docs/modules/kafka.asciidoc | 131 ++++++++++ metricbeat/docs/modules/kafka/broker.asciidoc | 24 ++ .../docs/modules/kafka/consumer.asciidoc | 24 ++ .../docs/modules/kafka/consumergroup.asciidoc | 22 ++ .../docs/modules/kafka/partition.asciidoc | 22 ++ .../docs/modules/kafka/producer.asciidoc | 24 ++ metricbeat/docs/modules/kibana.asciidoc | 57 +++++ metricbeat/docs/modules/kibana/stats.asciidoc | 21 ++ .../docs/modules/kibana/status.asciidoc | 22 ++ metricbeat/docs/modules/kubernetes.asciidoc | 235 ++++++++++++++++++ .../modules/kubernetes/apiserver.asciidoc | 21 ++ .../modules/kubernetes/container.asciidoc | 22 ++ .../kubernetes/controllermanager.asciidoc | 21 ++ .../docs/modules/kubernetes/event.asciidoc | 21 ++ .../docs/modules/kubernetes/node.asciidoc | 22 ++ .../docs/modules/kubernetes/pod.asciidoc | 22 ++ .../docs/modules/kubernetes/proxy.asciidoc | 21 ++ .../modules/kubernetes/scheduler.asciidoc | 21 ++ .../kubernetes/state_container.asciidoc | 21 ++ .../modules/kubernetes/state_cronjob.asciidoc | 21 ++ .../kubernetes/state_deployment.asciidoc | 21 ++ .../modules/kubernetes/state_node.asciidoc | 21 ++ .../modules/kubernetes/state_pod.asciidoc | 21 ++ .../kubernetes/state_replicaset.asciidoc | 21 ++ .../kubernetes/state_resourcequota.asciidoc | 21 ++ .../modules/kubernetes/state_service.asciidoc | 23 ++ .../kubernetes/state_statefulset.asciidoc | 21 ++ .../docs/modules/kubernetes/system.asciidoc | 22 ++ .../docs/modules/kubernetes/volume.asciidoc | 22 ++ metricbeat/docs/modules/kvm.asciidoc | 43 ++++ .../docs/modules/kvm/dommemstat.asciidoc | 24 ++ metricbeat/docs/modules/logstash.asciidoc | 49 ++++ .../docs/modules/logstash/node.asciidoc | 22 ++ .../docs/modules/logstash/node_stats.asciidoc | 22 ++ metricbeat/docs/modules/memcached.asciidoc | 42 ++++ .../docs/modules/memcached/stats.asciidoc | 22 ++ metricbeat/docs/modules/mongodb.asciidoc | 155 ++++++++++++ .../docs/modules/mongodb/collstats.asciidoc | 22 ++ .../docs/modules/mongodb/dbstats.asciidoc | 22 ++ .../docs/modules/mongodb/metrics.asciidoc | 22 ++ .../docs/modules/mongodb/replstatus.asciidoc | 21 ++ .../docs/modules/mongodb/status.asciidoc | 22 ++ metricbeat/docs/modules/mssql.asciidoc | 64 +++++ .../docs/modules/mssql/performance.asciidoc | 24 ++ .../modules/mssql/transaction_log.asciidoc | 24 ++ metricbeat/docs/modules/munin.asciidoc | 53 ++++ metricbeat/docs/modules/munin/node.asciidoc | 22 ++ metricbeat/docs/modules/mysql.asciidoc | 94 +++++++ .../docs/modules/mysql/galera_status.asciidoc | 23 ++ metricbeat/docs/modules/mysql/status.asciidoc | 22 ++ metricbeat/docs/modules/nats.asciidoc | 65 +++++ .../docs/modules/nats/connections.asciidoc | 22 ++ metricbeat/docs/modules/nats/routes.asciidoc | 22 ++ metricbeat/docs/modules/nats/stats.asciidoc | 22 ++ .../docs/modules/nats/subscriptions.asciidoc | 22 ++ metricbeat/docs/modules/nginx.asciidoc | 59 +++++ .../docs/modules/nginx/stubstatus.asciidoc | 22 ++ metricbeat/docs/modules/oracle.asciidoc | 81 ++++++ .../docs/modules/oracle/performance.asciidoc | 23 ++ .../docs/modules/oracle/tablespace.asciidoc | 23 ++ metricbeat/docs/modules/php_fpm.asciidoc | 75 ++++++ metricbeat/docs/modules/php_fpm/pool.asciidoc | 22 ++ .../docs/modules/php_fpm/process.asciidoc | 21 ++ metricbeat/docs/modules/postgresql.asciidoc | 124 +++++++++ .../docs/modules/postgresql/activity.asciidoc | 22 ++ .../docs/modules/postgresql/bgwriter.asciidoc | 22 ++ .../docs/modules/postgresql/database.asciidoc | 22 ++ .../modules/postgresql/statement.asciidoc | 21 ++ metricbeat/docs/modules/prometheus.asciidoc | 55 ++++ .../modules/prometheus/collector.asciidoc | 22 ++ metricbeat/docs/modules/rabbitmq.asciidoc | 66 +++++ .../docs/modules/rabbitmq/connection.asciidoc | 22 ++ .../docs/modules/rabbitmq/exchange.asciidoc | 22 ++ .../docs/modules/rabbitmq/node.asciidoc | 22 ++ .../docs/modules/rabbitmq/queue.asciidoc | 22 ++ metricbeat/docs/modules/redis.asciidoc | 97 ++++++++ metricbeat/docs/modules/redis/info.asciidoc | 22 ++ metricbeat/docs/modules/redis/key.asciidoc | 21 ++ .../docs/modules/redis/keyspace.asciidoc | 22 ++ metricbeat/docs/modules/statsd.asciidoc | 39 +++ .../docs/modules/statsd/server.asciidoc | 24 ++ metricbeat/docs/modules/system.asciidoc | 183 ++++++++++++++ metricbeat/docs/modules/system/core.asciidoc | 21 ++ metricbeat/docs/modules/system/cpu.asciidoc | 22 ++ .../docs/modules/system/diskio.asciidoc | 21 ++ .../docs/modules/system/entropy.asciidoc | 21 ++ .../docs/modules/system/filesystem.asciidoc | 21 ++ .../docs/modules/system/fsstat.asciidoc | 21 ++ metricbeat/docs/modules/system/load.asciidoc | 22 ++ .../docs/modules/system/memory.asciidoc | 22 ++ .../docs/modules/system/network.asciidoc | 22 ++ .../docs/modules/system/process.asciidoc | 22 ++ .../modules/system/process_summary.asciidoc | 22 ++ metricbeat/docs/modules/system/raid.asciidoc | 21 ++ .../docs/modules/system/service.asciidoc | 23 ++ .../docs/modules/system/socket.asciidoc | 21 ++ .../modules/system/socket_summary.asciidoc | 16 ++ .../docs/modules/system/uptime.asciidoc | 22 ++ metricbeat/docs/modules/tomcat.asciidoc | 58 +++++ metricbeat/docs/modules/tomcat/cache.asciidoc | 24 ++ .../docs/modules/tomcat/memory.asciidoc | 24 ++ .../docs/modules/tomcat/requests.asciidoc | 24 ++ .../docs/modules/tomcat/threading.asciidoc | 24 ++ metricbeat/docs/modules/traefik.asciidoc | 39 +++ .../docs/modules/traefik/health.asciidoc | 22 ++ metricbeat/docs/modules/uwsgi.asciidoc | 52 ++++ metricbeat/docs/modules/uwsgi/status.asciidoc | 22 ++ metricbeat/docs/modules/vsphere.asciidoc | 62 +++++ .../docs/modules/vsphere/datastore.asciidoc | 24 ++ metricbeat/docs/modules/vsphere/host.asciidoc | 24 ++ .../modules/vsphere/virtualmachine.asciidoc | 24 ++ metricbeat/docs/modules/windows.asciidoc | 51 ++++ .../docs/modules/windows/perfmon.asciidoc | 23 ++ .../docs/modules/windows/service.asciidoc | 21 ++ metricbeat/docs/modules/zookeeper.asciidoc | 57 +++++ .../modules/zookeeper/connection.asciidoc | 23 ++ .../docs/modules/zookeeper/mntr.asciidoc | 22 ++ .../docs/modules/zookeeper/server.asciidoc | 22 ++ metricbeat/include/list_common.go | 2 - metricbeat/metricbeat.reference.yml | 1 - metricbeat/module/iis/_meta/config.yml | 1 - metricbeat/module/iis/doc.go | 17 ++ metricbeat/module/iis/fields.go | 2 +- metricbeat/module/iis/module.yml | 4 + metricbeat/module/iis/performanceCounters.go | 17 ++ metricbeat/module/iis/reader.go | 3 +- .../module/iis/webserver/_meta/fields.yml | 6 - metricbeat/module/iis/webserver/manifest.yml | 12 + metricbeat/module/iis/webserver/webserver.go | 103 -------- .../module/iis/webserver/webserver_test.go | 18 ++ .../module/iis/website/_meta/fields.yml | 6 +- metricbeat/module/iis/website/manifest.yml | 11 + metricbeat/module/iis/website/website.go | 102 -------- metricbeat/module/iis/website/website_test.go | 18 ++ metricbeat/module/windows/perfmon/perfmon.go | 23 +- metricbeat/module/windows/perfmon/reader.go | 2 +- .../modules.d/{iis.yml => iis.yml.disabled} | 1 - .../{system.yml.disabled => system.yml} | 0 235 files changed, 7539 insertions(+), 249 deletions(-) create mode 100644 metricbeat/docs/modules/activemq.asciidoc create mode 100644 metricbeat/docs/modules/activemq/broker.asciidoc create mode 100644 metricbeat/docs/modules/activemq/queue.asciidoc create mode 100644 metricbeat/docs/modules/activemq/topic.asciidoc create mode 100644 metricbeat/docs/modules/aerospike.asciidoc create mode 100644 metricbeat/docs/modules/aerospike/namespace.asciidoc create mode 100644 metricbeat/docs/modules/apache.asciidoc create mode 100644 metricbeat/docs/modules/apache/status.asciidoc create mode 100644 metricbeat/docs/modules/appsearch.asciidoc create mode 100644 metricbeat/docs/modules/appsearch/stats.asciidoc create mode 100644 metricbeat/docs/modules/aws.asciidoc create mode 100644 metricbeat/docs/modules/aws/billing.asciidoc create mode 100644 metricbeat/docs/modules/aws/cloudwatch.asciidoc create mode 100644 metricbeat/docs/modules/aws/ebs.asciidoc create mode 100644 metricbeat/docs/modules/aws/ec2.asciidoc create mode 100644 metricbeat/docs/modules/aws/elb.asciidoc create mode 100644 metricbeat/docs/modules/aws/rds.asciidoc create mode 100644 metricbeat/docs/modules/aws/s3_daily_storage.asciidoc create mode 100644 metricbeat/docs/modules/aws/s3_request.asciidoc create mode 100644 metricbeat/docs/modules/aws/sns.asciidoc create mode 100644 metricbeat/docs/modules/aws/sqs.asciidoc create mode 100644 metricbeat/docs/modules/aws/usage.asciidoc create mode 100644 metricbeat/docs/modules/azure.asciidoc create mode 100644 metricbeat/docs/modules/azure/compute_vm.asciidoc create mode 100644 metricbeat/docs/modules/azure/compute_vm_scaleset.asciidoc create mode 100644 metricbeat/docs/modules/azure/monitor.asciidoc create mode 100644 metricbeat/docs/modules/beat.asciidoc create mode 100644 metricbeat/docs/modules/beat/state.asciidoc create mode 100644 metricbeat/docs/modules/beat/stats.asciidoc create mode 100644 metricbeat/docs/modules/ceph.asciidoc create mode 100644 metricbeat/docs/modules/ceph/cluster_disk.asciidoc create mode 100644 metricbeat/docs/modules/ceph/cluster_health.asciidoc create mode 100644 metricbeat/docs/modules/ceph/cluster_status.asciidoc create mode 100644 metricbeat/docs/modules/ceph/monitor_health.asciidoc create mode 100644 metricbeat/docs/modules/ceph/osd_df.asciidoc create mode 100644 metricbeat/docs/modules/ceph/osd_tree.asciidoc create mode 100644 metricbeat/docs/modules/ceph/pool_disk.asciidoc create mode 100644 metricbeat/docs/modules/cockroachdb.asciidoc create mode 100644 metricbeat/docs/modules/cockroachdb/status.asciidoc create mode 100644 metricbeat/docs/modules/consul.asciidoc create mode 100644 metricbeat/docs/modules/consul/agent.asciidoc create mode 100644 metricbeat/docs/modules/coredns.asciidoc create mode 100644 metricbeat/docs/modules/coredns/stats.asciidoc create mode 100644 metricbeat/docs/modules/couchbase.asciidoc create mode 100644 metricbeat/docs/modules/couchbase/bucket.asciidoc create mode 100644 metricbeat/docs/modules/couchbase/cluster.asciidoc create mode 100644 metricbeat/docs/modules/couchbase/node.asciidoc create mode 100644 metricbeat/docs/modules/couchdb.asciidoc create mode 100644 metricbeat/docs/modules/couchdb/server.asciidoc create mode 100644 metricbeat/docs/modules/docker.asciidoc create mode 100644 metricbeat/docs/modules/docker/container.asciidoc create mode 100644 metricbeat/docs/modules/docker/cpu.asciidoc create mode 100644 metricbeat/docs/modules/docker/diskio.asciidoc create mode 100644 metricbeat/docs/modules/docker/event.asciidoc create mode 100644 metricbeat/docs/modules/docker/healthcheck.asciidoc create mode 100644 metricbeat/docs/modules/docker/image.asciidoc create mode 100644 metricbeat/docs/modules/docker/info.asciidoc create mode 100644 metricbeat/docs/modules/docker/memory.asciidoc create mode 100644 metricbeat/docs/modules/docker/network.asciidoc create mode 100644 metricbeat/docs/modules/dropwizard.asciidoc create mode 100644 metricbeat/docs/modules/dropwizard/collector.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/ccr.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/cluster_stats.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/enrich.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/index.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/index_recovery.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/index_summary.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/ml_job.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/node.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/node_stats.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/pending_tasks.asciidoc create mode 100644 metricbeat/docs/modules/elasticsearch/shard.asciidoc create mode 100644 metricbeat/docs/modules/envoyproxy.asciidoc create mode 100644 metricbeat/docs/modules/envoyproxy/server.asciidoc create mode 100644 metricbeat/docs/modules/etcd.asciidoc create mode 100644 metricbeat/docs/modules/etcd/leader.asciidoc create mode 100644 metricbeat/docs/modules/etcd/metrics.asciidoc create mode 100644 metricbeat/docs/modules/etcd/self.asciidoc create mode 100644 metricbeat/docs/modules/etcd/store.asciidoc create mode 100644 metricbeat/docs/modules/golang.asciidoc create mode 100644 metricbeat/docs/modules/golang/expvar.asciidoc create mode 100644 metricbeat/docs/modules/golang/heap.asciidoc create mode 100644 metricbeat/docs/modules/graphite.asciidoc create mode 100644 metricbeat/docs/modules/graphite/server.asciidoc create mode 100644 metricbeat/docs/modules/haproxy.asciidoc create mode 100644 metricbeat/docs/modules/haproxy/info.asciidoc create mode 100644 metricbeat/docs/modules/haproxy/stat.asciidoc create mode 100644 metricbeat/docs/modules/http.asciidoc create mode 100644 metricbeat/docs/modules/http/json.asciidoc create mode 100644 metricbeat/docs/modules/http/server.asciidoc create mode 100644 metricbeat/docs/modules/iis.asciidoc create mode 100644 metricbeat/docs/modules/iis/webserver.asciidoc create mode 100644 metricbeat/docs/modules/iis/website.asciidoc create mode 100644 metricbeat/docs/modules/jolokia.asciidoc create mode 100644 metricbeat/docs/modules/jolokia/jmx.asciidoc create mode 100644 metricbeat/docs/modules/kafka.asciidoc create mode 100644 metricbeat/docs/modules/kafka/broker.asciidoc create mode 100644 metricbeat/docs/modules/kafka/consumer.asciidoc create mode 100644 metricbeat/docs/modules/kafka/consumergroup.asciidoc create mode 100644 metricbeat/docs/modules/kafka/partition.asciidoc create mode 100644 metricbeat/docs/modules/kafka/producer.asciidoc create mode 100644 metricbeat/docs/modules/kibana.asciidoc create mode 100644 metricbeat/docs/modules/kibana/stats.asciidoc create mode 100644 metricbeat/docs/modules/kibana/status.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/apiserver.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/container.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/controllermanager.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/event.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/node.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/pod.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/proxy.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/scheduler.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_container.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_cronjob.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_deployment.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_node.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_pod.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_replicaset.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_resourcequota.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_service.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/state_statefulset.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/system.asciidoc create mode 100644 metricbeat/docs/modules/kubernetes/volume.asciidoc create mode 100644 metricbeat/docs/modules/kvm.asciidoc create mode 100644 metricbeat/docs/modules/kvm/dommemstat.asciidoc create mode 100644 metricbeat/docs/modules/logstash.asciidoc create mode 100644 metricbeat/docs/modules/logstash/node.asciidoc create mode 100644 metricbeat/docs/modules/logstash/node_stats.asciidoc create mode 100644 metricbeat/docs/modules/memcached.asciidoc create mode 100644 metricbeat/docs/modules/memcached/stats.asciidoc create mode 100644 metricbeat/docs/modules/mongodb.asciidoc create mode 100644 metricbeat/docs/modules/mongodb/collstats.asciidoc create mode 100644 metricbeat/docs/modules/mongodb/dbstats.asciidoc create mode 100644 metricbeat/docs/modules/mongodb/metrics.asciidoc create mode 100644 metricbeat/docs/modules/mongodb/replstatus.asciidoc create mode 100644 metricbeat/docs/modules/mongodb/status.asciidoc create mode 100644 metricbeat/docs/modules/mssql.asciidoc create mode 100644 metricbeat/docs/modules/mssql/performance.asciidoc create mode 100644 metricbeat/docs/modules/mssql/transaction_log.asciidoc create mode 100644 metricbeat/docs/modules/munin.asciidoc create mode 100644 metricbeat/docs/modules/munin/node.asciidoc create mode 100644 metricbeat/docs/modules/mysql.asciidoc create mode 100644 metricbeat/docs/modules/mysql/galera_status.asciidoc create mode 100644 metricbeat/docs/modules/mysql/status.asciidoc create mode 100644 metricbeat/docs/modules/nats.asciidoc create mode 100644 metricbeat/docs/modules/nats/connections.asciidoc create mode 100644 metricbeat/docs/modules/nats/routes.asciidoc create mode 100644 metricbeat/docs/modules/nats/stats.asciidoc create mode 100644 metricbeat/docs/modules/nats/subscriptions.asciidoc create mode 100644 metricbeat/docs/modules/nginx.asciidoc create mode 100644 metricbeat/docs/modules/nginx/stubstatus.asciidoc create mode 100644 metricbeat/docs/modules/oracle.asciidoc create mode 100644 metricbeat/docs/modules/oracle/performance.asciidoc create mode 100644 metricbeat/docs/modules/oracle/tablespace.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm/pool.asciidoc create mode 100644 metricbeat/docs/modules/php_fpm/process.asciidoc create mode 100644 metricbeat/docs/modules/postgresql.asciidoc create mode 100644 metricbeat/docs/modules/postgresql/activity.asciidoc create mode 100644 metricbeat/docs/modules/postgresql/bgwriter.asciidoc create mode 100644 metricbeat/docs/modules/postgresql/database.asciidoc create mode 100644 metricbeat/docs/modules/postgresql/statement.asciidoc create mode 100644 metricbeat/docs/modules/prometheus.asciidoc create mode 100644 metricbeat/docs/modules/prometheus/collector.asciidoc create mode 100644 metricbeat/docs/modules/rabbitmq.asciidoc create mode 100644 metricbeat/docs/modules/rabbitmq/connection.asciidoc create mode 100644 metricbeat/docs/modules/rabbitmq/exchange.asciidoc create mode 100644 metricbeat/docs/modules/rabbitmq/node.asciidoc create mode 100644 metricbeat/docs/modules/rabbitmq/queue.asciidoc create mode 100644 metricbeat/docs/modules/redis.asciidoc create mode 100644 metricbeat/docs/modules/redis/info.asciidoc create mode 100644 metricbeat/docs/modules/redis/key.asciidoc create mode 100644 metricbeat/docs/modules/redis/keyspace.asciidoc create mode 100644 metricbeat/docs/modules/statsd.asciidoc create mode 100644 metricbeat/docs/modules/statsd/server.asciidoc create mode 100644 metricbeat/docs/modules/system.asciidoc create mode 100644 metricbeat/docs/modules/system/core.asciidoc create mode 100644 metricbeat/docs/modules/system/cpu.asciidoc create mode 100644 metricbeat/docs/modules/system/diskio.asciidoc create mode 100644 metricbeat/docs/modules/system/entropy.asciidoc create mode 100644 metricbeat/docs/modules/system/filesystem.asciidoc create mode 100644 metricbeat/docs/modules/system/fsstat.asciidoc create mode 100644 metricbeat/docs/modules/system/load.asciidoc create mode 100644 metricbeat/docs/modules/system/memory.asciidoc create mode 100644 metricbeat/docs/modules/system/network.asciidoc create mode 100644 metricbeat/docs/modules/system/process.asciidoc create mode 100644 metricbeat/docs/modules/system/process_summary.asciidoc create mode 100644 metricbeat/docs/modules/system/raid.asciidoc create mode 100644 metricbeat/docs/modules/system/service.asciidoc create mode 100644 metricbeat/docs/modules/system/socket.asciidoc create mode 100644 metricbeat/docs/modules/system/socket_summary.asciidoc create mode 100644 metricbeat/docs/modules/system/uptime.asciidoc create mode 100644 metricbeat/docs/modules/tomcat.asciidoc create mode 100644 metricbeat/docs/modules/tomcat/cache.asciidoc create mode 100644 metricbeat/docs/modules/tomcat/memory.asciidoc create mode 100644 metricbeat/docs/modules/tomcat/requests.asciidoc create mode 100644 metricbeat/docs/modules/tomcat/threading.asciidoc create mode 100644 metricbeat/docs/modules/traefik.asciidoc create mode 100644 metricbeat/docs/modules/traefik/health.asciidoc create mode 100644 metricbeat/docs/modules/uwsgi.asciidoc create mode 100644 metricbeat/docs/modules/uwsgi/status.asciidoc create mode 100644 metricbeat/docs/modules/vsphere.asciidoc create mode 100644 metricbeat/docs/modules/vsphere/datastore.asciidoc create mode 100644 metricbeat/docs/modules/vsphere/host.asciidoc create mode 100644 metricbeat/docs/modules/vsphere/virtualmachine.asciidoc create mode 100644 metricbeat/docs/modules/windows.asciidoc create mode 100644 metricbeat/docs/modules/windows/perfmon.asciidoc create mode 100644 metricbeat/docs/modules/windows/service.asciidoc create mode 100644 metricbeat/docs/modules/zookeeper.asciidoc create mode 100644 metricbeat/docs/modules/zookeeper/connection.asciidoc create mode 100644 metricbeat/docs/modules/zookeeper/mntr.asciidoc create mode 100644 metricbeat/docs/modules/zookeeper/server.asciidoc create mode 100644 metricbeat/module/iis/module.yml create mode 100644 metricbeat/module/iis/webserver/manifest.yml delete mode 100644 metricbeat/module/iis/webserver/webserver.go create mode 100644 metricbeat/module/iis/webserver/webserver_test.go create mode 100644 metricbeat/module/iis/website/manifest.yml delete mode 100644 metricbeat/module/iis/website/website.go create mode 100644 metricbeat/module/iis/website/website_test.go rename metricbeat/modules.d/{iis.yml => iis.yml.disabled} (92%) rename metricbeat/modules.d/{system.yml.disabled => system.yml} (100%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3e14338bf011..4bb9b6de36d9 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -15283,34 +15283,12 @@ iis module webserver - -*`iis.webserver.example`*:: -+ --- -Example field - - -type: keyword - --- - [float] === website website - -*`iis.website.example`*:: -+ --- -Example field - - -type: keyword - --- - [[exported-fields-jolokia]] == Jolokia fields diff --git a/metricbeat/docs/modules/activemq.asciidoc b/metricbeat/docs/modules/activemq.asciidoc new file mode 100644 index 000000000000..3bd03addf9cc --- /dev/null +++ b/metricbeat/docs/modules/activemq.asciidoc @@ -0,0 +1,86 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-activemq]] +[role="xpack"] +== activemq module + +beta[] + +This module periodically fetches JMX metrics from Apache ActiveMQ. + +[float] +=== Compatibility +The module has been tested with ActiveMQ 5.13.0 and 5.15.9. Other versions are expected to work. + +[float] +=== Usage +The ActiveMQ module requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. + + +[float] +=== Example configuration + +The activemq module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: activemq + metricsets: ['broker', 'queue', 'topic'] + period: 10s + hosts: ['localhost:8161'] + path: '/api/jolokia/?ignoreErrors=true&canonicalNaming=false' + username: admin # default username + password: admin # default password + processors: + - script: + lang: javascript + source: > + function process(event) { + var broker_memory_broker_pct = event.Get("activemq.broker.memory.broker.pct") + if (broker_memory_broker_pct != null) { + event.Put("activemq.broker.memory.broker.pct", broker_memory_broker_pct / 100.0) + } + + var broker_memory_temp_pct = event.Get("activemq.broker.memory.temp.pct") + if (broker_memory_temp_pct != null) { + event.Put("activemq.broker.memory.temp.pct", broker_memory_temp_pct / 100.0) + } + + var broker_memory_store_pct = event.Get("activemq.broker.memory.store.pct") + if (broker_memory_store_pct != null) { + event.Put("activemq.broker.memory.store.pct", broker_memory_store_pct / 100.0) + } + + var queue_memory_broker_pct = event.Get("activemq.queue.memory.broker.pct") + if (queue_memory_broker_pct != null) { + event.Put("activemq.queue.memory.broker.pct", queue_memory_broker_pct / 100.0) + } + + var topic_memory_broker_pct = event.Get("activemq.topic.memory.broker.pct") + if (topic_memory_broker_pct != null) { + event.Put("activemq.topic.memory.broker.pct", topic_memory_broker_pct / 100.0) + } + } +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::activemq/broker.asciidoc[] + +include::activemq/queue.asciidoc[] + +include::activemq/topic.asciidoc[] + diff --git a/metricbeat/docs/modules/activemq/broker.asciidoc b/metricbeat/docs/modules/activemq/broker.asciidoc new file mode 100644 index 000000000000..16e5cd8f4a8f --- /dev/null +++ b/metricbeat/docs/modules/activemq/broker.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-activemq-broker]] +=== activemq broker metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/activemq/broker/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/activemq/broker/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/activemq/queue.asciidoc b/metricbeat/docs/modules/activemq/queue.asciidoc new file mode 100644 index 000000000000..84f0e17fffdf --- /dev/null +++ b/metricbeat/docs/modules/activemq/queue.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-activemq-queue]] +=== activemq queue metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/activemq/queue/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/activemq/queue/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/activemq/topic.asciidoc b/metricbeat/docs/modules/activemq/topic.asciidoc new file mode 100644 index 000000000000..fec92cbf9ccf --- /dev/null +++ b/metricbeat/docs/modules/activemq/topic.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-activemq-topic]] +=== activemq topic metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/activemq/topic/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/activemq/topic/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aerospike.asciidoc b/metricbeat/docs/modules/aerospike.asciidoc new file mode 100644 index 000000000000..f29f7382a8c7 --- /dev/null +++ b/metricbeat/docs/modules/aerospike.asciidoc @@ -0,0 +1,48 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-aerospike]] +== Aerospike module + +The Aerospike module uses the http://www.aerospike.com/docs/reference/info[Info command] to collect metrics. The default metricset is `namespace`. + +[float] +=== Compatibility + +The Aerospike metricsets were tested with Aerospike 3.9 and are expected to work with all versions >= 3.9. + + +[float] +=== Dashboard + +The Aerospike module comes with a predefined dashboard for Aerospike namespace, node specific stats. For example: + +image::./images/metricbeat-aerospike-overview.png[] + + +[float] +=== Example configuration + +The Aerospike module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: aerospike + metricsets: ["namespace"] + enabled: true + period: 10s + hosts: ["localhost:3000"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::aerospike/namespace.asciidoc[] + diff --git a/metricbeat/docs/modules/aerospike/namespace.asciidoc b/metricbeat/docs/modules/aerospike/namespace.asciidoc new file mode 100644 index 000000000000..47af311c1ddd --- /dev/null +++ b/metricbeat/docs/modules/aerospike/namespace.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aerospike-namespace]] +=== Aerospike namespace metricset + +include::../../../module/aerospike/namespace/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/aerospike/namespace/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/apache.asciidoc b/metricbeat/docs/modules/apache.asciidoc new file mode 100644 index 000000000000..e8f6028f3a7e --- /dev/null +++ b/metricbeat/docs/modules/apache.asciidoc @@ -0,0 +1,64 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-apache]] +== Apache module + +This module periodically fetches metrics from https://httpd.apache.org/[Apache +HTTPD] servers. The default metricset is `status`. + +[float] +=== Compatibility + +The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16. + + +[float] +=== Dashboard + +The Apache module comes with a predefined dashboard. For example: + +image::./images/apache_httpd_server_status.png[] + + +[float] +=== Example configuration + +The Apache module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: apache + metricsets: ["status"] + period: 10s + enabled: true + + # Apache hosts + hosts: ["http://127.0.0.1"] + + # Path to server status. Default server-status + #server_status_path: "server-status" + + # Username of hosts. Empty by default + #username: username + + # Password of hosts. Empty by default + #password: password +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::apache/status.asciidoc[] + diff --git a/metricbeat/docs/modules/apache/status.asciidoc b/metricbeat/docs/modules/apache/status.asciidoc new file mode 100644 index 000000000000..35981eabd5ba --- /dev/null +++ b/metricbeat/docs/modules/apache/status.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-apache-status]] +=== Apache status metricset + +include::../../../module/apache/status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/apache/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/appsearch.asciidoc b/metricbeat/docs/modules/appsearch.asciidoc new file mode 100644 index 000000000000..69a2b1da473f --- /dev/null +++ b/metricbeat/docs/modules/appsearch.asciidoc @@ -0,0 +1,41 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-appsearch]] +[role="xpack"] +== App Search module + +beta[] + +This is the App Search module. + + + +[float] +=== Example configuration + +The App Search module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: appsearch + metricsets: ["stats"] + enabled: true + period: 10s + hosts: ["http://localhost:3002"] + #username: "elastic" + #password: "changeme" +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::appsearch/stats.asciidoc[] + diff --git a/metricbeat/docs/modules/appsearch/stats.asciidoc b/metricbeat/docs/modules/appsearch/stats.asciidoc new file mode 100644 index 000000000000..e9bcccbed769 --- /dev/null +++ b/metricbeat/docs/modules/appsearch/stats.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-appsearch-stats]] +=== App Search stats metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/appsearch/stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/appsearch/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc new file mode 100644 index 000000000000..a23d09cb28e7 --- /dev/null +++ b/metricbeat/docs/modules/aws.asciidoc @@ -0,0 +1,195 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-aws]] +[role="xpack"] +== aws module + +:libbeat-xpack-dir: ../../../x-pack/libbeat + +This module periodically fetches monitoring metrics from AWS Cloudwatch using +https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services. +Note: extra AWS charges on GetMetricData API requests will be generated by this module. + +The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage`, `cloudwatch` and `rds`. + +[float] +=== Module-specific configuration notes + +The `aws` module requires AWS credentials configuration in order to make AWS API calls. +Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or +`AWS_SESSION_TOKEN`, or use shared AWS credentials file. +Please see <> for more details. + +This module also accepts optional configuration `regions` to specify which +AWS regions to query metrics from. If the `regions` parameter is not set in the +config file, then by default, the `aws` module will query metrics from all available +AWS regions. + +The aws module comes with a predefined dashboard. For example: + +image::./images/metricbeat-aws-overview.png[] + +[float] +== Metricsets + +Currently, we have `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch` metricset in `aws` module. Collecting `tags` +for `ec2` and `cloudwatch` metricset is supported. + +* *tags.*: Tag key value pairs from aws resources. A tag is a label that user assigns to an AWS resource. + +[float] +=== `ec2` +By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module +configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics +will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs +$0.01/1000 metrics requested using GetMetricData. Please see https://aws.amazon.com/cloudwatch/pricing/[AWS Cloudwatch Pricing] +for more details. To avoid unnecessary charges, `period` is preferred to be set to `300s` or multiples of `300s`, such as +`600s` and `900s`. For more granular monitoring data you can enable detailed monitoring on the instance to get metrics every 1 minute. Please see +https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html[Enabling Detailed Monitoring] for instructions +on how to enable detailed monitoring. With detailed monitoring enabled, `period` in aws module configuration can be any number +larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods, setting metricbeat module `period` less +than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is +preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. + +The ec2 metricset comes with a predefined dashboard. For example: + +image::./images/metricbeat-aws-ec2-overview.png[] + +[float] +=== `sqs` +Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, +the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. + +[float] +=== `s3_daily_storage` +Daily storage metrics for S3 buckets are reported once per day with no additional cost. Since they are daily metrics, +`period` for `s3_daily_storage` metricset is recommended to be `86400s` or multiples of `86400s`. + +[float] +=== `s3_request` +Request metrics are available +at 1-minute intervals with additional charges. The s3_request metricset will give more +granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. +But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can +always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see +https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to +Configure Request Metrics for S3] for instructions on how to enable request metrics for +each S3 bucket. + +[float] +=== `cloudwatch` +This metricset gives users the freedom to query metrics from AWS Cloudwatch with +any given namespaces or specific instance with a given period. +Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/aws-services-cloudwatch-metrics.html[AWS Services That Publish CloudWatch Metrics] +for a list of AWS services that publish metrics to CloudWatch. + +[float] +=== `rds` +`period` for `rds` metricset is recommended to be `60s` or multiples of `60s` because Amazon RDS sends metrics and +dimensions to Amazon CloudWatch every minute. + +[id="aws-credentials-config"] +include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] + + +[float] +=== Example configuration + +The aws module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: aws + period: 300s + credential_profile_name: test-mb + metricsets: + - ec2 +- module: aws + period: 300s + credential_profile_name: test-mb + metricsets: + - sqs + regions: + - us-west-1 +- module: aws + period: 86400s + metricsets: + - s3_request + - s3_daily_storage + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' +- module: aws + period: 300s + credential_profile_name: test-mb + metricsets: + - cloudwatch + metrics: + - namespace: AWS/EC2 + name: ["CPUUtilization"] + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS + - namespace: AWS/ELB + tags.resource_type_filter: elasticloadbalancing +- module: aws + period: 60s + credential_profile_name: test-mb + metricsets: + - rds +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +include::aws/billing.asciidoc[] + +include::aws/cloudwatch.asciidoc[] + +include::aws/ebs.asciidoc[] + +include::aws/ec2.asciidoc[] + +include::aws/elb.asciidoc[] + +include::aws/rds.asciidoc[] + +include::aws/s3_daily_storage.asciidoc[] + +include::aws/s3_request.asciidoc[] + +include::aws/sns.asciidoc[] + +include::aws/sqs.asciidoc[] + +include::aws/usage.asciidoc[] + diff --git a/metricbeat/docs/modules/aws/billing.asciidoc b/metricbeat/docs/modules/aws/billing.asciidoc new file mode 100644 index 000000000000..6db84ce54539 --- /dev/null +++ b/metricbeat/docs/modules/aws/billing.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-billing]] +=== aws billing metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/billing/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/billing/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/cloudwatch.asciidoc b/metricbeat/docs/modules/aws/cloudwatch.asciidoc new file mode 100644 index 000000000000..7fb2a68ea4d6 --- /dev/null +++ b/metricbeat/docs/modules/aws/cloudwatch.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-cloudwatch]] +=== aws cloudwatch metricset + +include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/ebs.asciidoc b/metricbeat/docs/modules/aws/ebs.asciidoc new file mode 100644 index 000000000000..73613b8b0b80 --- /dev/null +++ b/metricbeat/docs/modules/aws/ebs.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-ebs]] +=== aws ebs metricset + +include::../../../../x-pack/metricbeat/module/aws/ebs/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/ebs/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/ec2.asciidoc b/metricbeat/docs/modules/aws/ec2.asciidoc new file mode 100644 index 000000000000..271b95987d6d --- /dev/null +++ b/metricbeat/docs/modules/aws/ec2.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-ec2]] +=== aws ec2 metricset + +include::../../../../x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/ec2/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/elb.asciidoc b/metricbeat/docs/modules/aws/elb.asciidoc new file mode 100644 index 000000000000..dda9ad02e8e2 --- /dev/null +++ b/metricbeat/docs/modules/aws/elb.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-elb]] +=== aws elb metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/elb/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/elb/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/rds.asciidoc b/metricbeat/docs/modules/aws/rds.asciidoc new file mode 100644 index 000000000000..c4564faaf960 --- /dev/null +++ b/metricbeat/docs/modules/aws/rds.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-rds]] +=== aws rds metricset + +include::../../../../x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/rds/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc new file mode 100644 index 000000000000..de7a74d7e8b4 --- /dev/null +++ b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-s3_daily_storage]] +=== aws s3_daily_storage metricset + +include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/s3_request.asciidoc b/metricbeat/docs/modules/aws/s3_request.asciidoc new file mode 100644 index 000000000000..131a720589de --- /dev/null +++ b/metricbeat/docs/modules/aws/s3_request.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-s3_request]] +=== aws s3_request metricset + +include::../../../../x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/s3_request/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/sns.asciidoc b/metricbeat/docs/modules/aws/sns.asciidoc new file mode 100644 index 000000000000..dbc1c0a31605 --- /dev/null +++ b/metricbeat/docs/modules/aws/sns.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-sns]] +=== aws sns metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/sns/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/sns/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/sqs.asciidoc b/metricbeat/docs/modules/aws/sqs.asciidoc new file mode 100644 index 000000000000..8fa5a2120fb2 --- /dev/null +++ b/metricbeat/docs/modules/aws/sqs.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-sqs]] +=== aws sqs metricset + +include::../../../../x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/sqs/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/aws/usage.asciidoc b/metricbeat/docs/modules/aws/usage.asciidoc new file mode 100644 index 000000000000..5258fab87bc2 --- /dev/null +++ b/metricbeat/docs/modules/aws/usage.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-aws-usage]] +=== aws usage metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/usage/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/usage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc new file mode 100644 index 000000000000..b37998ad191d --- /dev/null +++ b/metricbeat/docs/modules/azure.asciidoc @@ -0,0 +1,151 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-azure]] +[role="xpack"] +== azure module + +beta[] + +This is the azure module. + +The Azure Monitor feature collects and aggregates logs and metrics from a variety of sources into a common data platform where it can be used for analysis, visualization, and alerting. + + +The azure monitor metrics are numerical values that describe some aspect of a system at a particular point in time. They are collected at regular intervals and are identified with a timestamp, a name, a value, and one or more defining labels. + +The azure module will periodically retrieve the azure monitor metrics using the Azure REST APIs as MetricList. +Additional azure API calls will be executed in order to retrieve information regarding the resources targeted by the user. + +The azure module mericsets are `monitor`, `compute_vm` and `compute_vm_scaleset` + +[float] +=== Dashboards + +The azure module comes with several predefined dashboards for virtual machines, VM guest metrics and virtual machine scale sets. + +The VM overview dashboard shows information about CPU, memory, disk usage as well as operations per second. The two available filters help narrowing down the dashbord to specific regions and/or resource groups. For example: + +image::./images/metricbeat-azure-vm-overview.png[] + +If VM guest metrics are enabled then the guest metrics overview dashboard can help with monitoring ASP.NET applications and SQL Server metrics. For example: + +image::./images/metricbeat-azure-vm-guestmetrics-overview.png[] + +The virtual machine scale sets dashboard is similar to the VM dashboard and shows relevant health information about running vm scale sets. For example: + +image::./images/metricbeat-azure-vmss-overview.png[] + +[float] +=== Module-specific configuration notes + +All the tasks executed against the Azure Monitor REST API will use the Azure Resource Manager authentication model. +Therefore, all requests must be authenticated with Azure Active Directory (Azure AD). +One approach to authenticate the client application is to create an Azure AD service principal and retrieve the authentication (JWT) token. +For a more detailed walk-through, have a look at using Azure PowerShell to create a service principal to access resources https://docs.microsoft.com/en-us/powershell/azure/create-azure-service-principal-azureps?view=azps-2.7.0. + It is also possible to create a service principal via the Azure portal https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal. +Users will have to make sure the roles assigned to the application contain at least reading permissions to the monitor data, more on the roles here https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles. + +Required credentials for the `azure` module: + +`client_id`:: The unique identifier for the application (also known as Application Id) + +`client_secret`:: The client/application secret/key + +`subscription_id`:: The unique identifier for the azure subscription + +`tenant_id`:: The unique identifier of the Azure Active Directory instance + + +Users can use the azure credentials keys if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` + +[float] +== Metricsets + +[float] +=== `monitor` +This metricset allows users to retrieve metrics from specified resources. Added filters can apply here as the interval of retrieving these metrics, metric names, +aggregation list, namespaces and metric dimensions. + +[float] +=== `compute_vm` +This metricset will collect metrics from the virtual machines, these metrics will have a timegrain every 5 minutes, +so the `period` for `compute_vm` metricset should be `300s` or multiples of `300s`. + +[float] +=== `compute_vm_scaleset` +This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes, +so the `period` for `compute_vm_scaleset` metricset should be `300s` or multiples of `300s`. + + +[float] +== Additional notes about metrics and costs + +Costs: Metric queries are charged based on the number of standard API calls. More information on pricing here https://azure.microsoft.com/id-id/pricing/details/monitor/. + +Authentication: we are handling authentication on our side (creating/renewing the authentication token), so we advise users to use dedicated credentials for metricbeat only. + + +[float] +=== Example configuration + +The azure module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: azure + metricsets: + - monitor + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + resources: + - resource_query: "resourceType eq 'Microsoft.DocumentDb/databaseAccounts'" + metrics: + - name: ["DataUsage", "DocumentCount", "DocumentQuota"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + +- module: azure + metricsets: + - compute_vm + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - compute_vm_scaleset + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::azure/compute_vm.asciidoc[] + +include::azure/compute_vm_scaleset.asciidoc[] + +include::azure/monitor.asciidoc[] + diff --git a/metricbeat/docs/modules/azure/compute_vm.asciidoc b/metricbeat/docs/modules/azure/compute_vm.asciidoc new file mode 100644 index 000000000000..39d3be6eae09 --- /dev/null +++ b/metricbeat/docs/modules/azure/compute_vm.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-compute_vm]] +=== azure compute_vm metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/compute_vm/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure/compute_vm_scaleset.asciidoc b/metricbeat/docs/modules/azure/compute_vm_scaleset.asciidoc new file mode 100644 index 000000000000..0bc72248c53c --- /dev/null +++ b/metricbeat/docs/modules/azure/compute_vm_scaleset.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-compute_vm_scaleset]] +=== azure compute_vm_scaleset metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure/monitor.asciidoc b/metricbeat/docs/modules/azure/monitor.asciidoc new file mode 100644 index 000000000000..1596987e18ec --- /dev/null +++ b/metricbeat/docs/modules/azure/monitor.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-monitor]] +=== azure monitor metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/monitor/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/beat.asciidoc b/metricbeat/docs/modules/beat.asciidoc new file mode 100644 index 000000000000..bfa9667b109e --- /dev/null +++ b/metricbeat/docs/modules/beat.asciidoc @@ -0,0 +1,55 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-beat]] +== Beat module + +This is the Beat module. + +The default metricsets are `state` and `stats`. + +[float] +=== Compatibility + +The beat module is tested with metricbeat 7.3.0. + + +[float] +=== Example configuration + +The Beat module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: beat + metricsets: + - stats + - state + period: 10s + hosts: ["http://localhost:5066"] + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::beat/state.asciidoc[] + +include::beat/stats.asciidoc[] + diff --git a/metricbeat/docs/modules/beat/state.asciidoc b/metricbeat/docs/modules/beat/state.asciidoc new file mode 100644 index 000000000000..b32fbf007b35 --- /dev/null +++ b/metricbeat/docs/modules/beat/state.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-beat-state]] +=== Beat state metricset + +include::../../../module/beat/state/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/beat/state/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/beat/stats.asciidoc b/metricbeat/docs/modules/beat/stats.asciidoc new file mode 100644 index 000000000000..483192899365 --- /dev/null +++ b/metricbeat/docs/modules/beat/stats.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-beat-stats]] +=== Beat stats metricset + +include::../../../module/beat/stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/beat/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc new file mode 100644 index 000000000000..1fb296f1784c --- /dev/null +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -0,0 +1,76 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-ceph]] +== Ceph module + +The Ceph module collects metrics by submitting HTTP GET requests to +the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The default metricsets are `cluster_disk`, `cluster_health`, `monitor_health`, `pool_disk`, `osd_tree`. + +[float] +=== Compatibility + +The Ceph module is tested with Ceph Jewel (10.2.10). + +[float] +=== Dashboard + +The Ceph module comes with a predefined dashboard showing Ceph cluster related metrics. For example: + +image::./images/ceph-overview-dashboard.png[] + + + +[float] +=== Example configuration + +The Ceph module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: ceph + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] + period: 10s + hosts: ["localhost:5000"] + enabled: true +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +include::ceph/cluster_disk.asciidoc[] + +include::ceph/cluster_health.asciidoc[] + +include::ceph/cluster_status.asciidoc[] + +include::ceph/monitor_health.asciidoc[] + +include::ceph/osd_df.asciidoc[] + +include::ceph/osd_tree.asciidoc[] + +include::ceph/pool_disk.asciidoc[] + diff --git a/metricbeat/docs/modules/ceph/cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/cluster_disk.asciidoc new file mode 100644 index 000000000000..e6c320602772 --- /dev/null +++ b/metricbeat/docs/modules/ceph/cluster_disk.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-cluster_disk]] +=== Ceph cluster_disk metricset + +include::../../../module/ceph/cluster_disk/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/cluster_disk/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/cluster_health.asciidoc b/metricbeat/docs/modules/ceph/cluster_health.asciidoc new file mode 100644 index 000000000000..6614f8369e21 --- /dev/null +++ b/metricbeat/docs/modules/ceph/cluster_health.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-cluster_health]] +=== Ceph cluster_health metricset + +include::../../../module/ceph/cluster_health/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/cluster_health/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/cluster_status.asciidoc b/metricbeat/docs/modules/ceph/cluster_status.asciidoc new file mode 100644 index 000000000000..85fd091476b9 --- /dev/null +++ b/metricbeat/docs/modules/ceph/cluster_status.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-cluster_status]] +=== Ceph cluster_status metricset + +include::../../../module/ceph/cluster_status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/cluster_status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/monitor_health.asciidoc b/metricbeat/docs/modules/ceph/monitor_health.asciidoc new file mode 100644 index 000000000000..98e9671aecd7 --- /dev/null +++ b/metricbeat/docs/modules/ceph/monitor_health.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-monitor_health]] +=== Ceph monitor_health metricset + +include::../../../module/ceph/monitor_health/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/monitor_health/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/osd_df.asciidoc b/metricbeat/docs/modules/ceph/osd_df.asciidoc new file mode 100644 index 000000000000..a99ecb258eb9 --- /dev/null +++ b/metricbeat/docs/modules/ceph/osd_df.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-osd_df]] +=== Ceph osd_df metricset + +include::../../../module/ceph/osd_df/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/osd_df/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/osd_tree.asciidoc b/metricbeat/docs/modules/ceph/osd_tree.asciidoc new file mode 100644 index 000000000000..91ad066655ad --- /dev/null +++ b/metricbeat/docs/modules/ceph/osd_tree.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-osd_tree]] +=== Ceph osd_tree metricset + +include::../../../module/ceph/osd_tree/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/osd_tree/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/pool_disk.asciidoc b/metricbeat/docs/modules/ceph/pool_disk.asciidoc new file mode 100644 index 000000000000..12afb1c2f42f --- /dev/null +++ b/metricbeat/docs/modules/ceph/pool_disk.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-pool_disk]] +=== Ceph pool_disk metricset + +include::../../../module/ceph/pool_disk/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/pool_disk/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/cockroachdb.asciidoc b/metricbeat/docs/modules/cockroachdb.asciidoc new file mode 100644 index 000000000000..27050f77b8c8 --- /dev/null +++ b/metricbeat/docs/modules/cockroachdb.asciidoc @@ -0,0 +1,59 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-cockroachdb]] +[role="xpack"] +== CockroachDB module + +beta[] + +This module periodically fetches metrics from CockroachDB. + +[float] +=== Compatibility + +The CockroachDB `status` metricset is compatible with any CockroachDB version +exposing metrics in Prometheus format. + + +[float] +=== Dashboard + +The CockroachDB module includes a predefined dashboard with overview information +of the monitored servers. + +image::./images/metricbeat-cockroachdb-overview.png[] + + +[float] +=== Example configuration + +The CockroachDB module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: cockroachdb + metricsets: ['status'] + period: 10s + hosts: ['localhost:8080'] + + # This module uses the Prometheus collector metricset, all + # the options for this metricset are also available here. + #metrics_path: /_status/vars +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::cockroachdb/status.asciidoc[] + diff --git a/metricbeat/docs/modules/cockroachdb/status.asciidoc b/metricbeat/docs/modules/cockroachdb/status.asciidoc new file mode 100644 index 000000000000..d15dd6fc0b88 --- /dev/null +++ b/metricbeat/docs/modules/cockroachdb/status.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-cockroachdb-status]] +=== CockroachDB status metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/cockroachdb/status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/cockroachdb/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/consul.asciidoc b/metricbeat/docs/modules/consul.asciidoc new file mode 100644 index 000000000000..a36b9b0a5890 --- /dev/null +++ b/metricbeat/docs/modules/consul.asciidoc @@ -0,0 +1,51 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-consul]] +== consul module + +beta[] + +This is the https://www.consul.io[Hashicorp's Consul] Metricbeat module. It is still in beta and under active development to add new Metricsets and introduce enhancements. + +[float] +=== Compatibility + +The module is being tested with https://github.com/hashicorp/docker-consul/blob/9bd2aa7ecf2414b8712e055f2374699148e8941c/0.X/Dockerfile[1.4.2] version + +[float] +=== Dashboard + +The Consul module comes with a predefined dashboard: + +image::./images/metricbeat-consul.png[] + + +[float] +=== Example configuration + +The consul module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: consul + metricsets: + - agent + enabled: true + period: 10s + hosts: ["localhost:8500"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::consul/agent.asciidoc[] + diff --git a/metricbeat/docs/modules/consul/agent.asciidoc b/metricbeat/docs/modules/consul/agent.asciidoc new file mode 100644 index 000000000000..caef941ef551 --- /dev/null +++ b/metricbeat/docs/modules/consul/agent.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-consul-agent]] +=== consul agent metricset + +beta[] + +include::../../../module/consul/agent/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/consul/agent/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/coredns.asciidoc b/metricbeat/docs/modules/coredns.asciidoc new file mode 100644 index 000000000000..cfe8ff6144ef --- /dev/null +++ b/metricbeat/docs/modules/coredns.asciidoc @@ -0,0 +1,51 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-coredns]] +[role="xpack"] +== coredns module + +This is the CoreDNS module. The CoreDNS module collects metrics from the +CoreDNS https://github.com/coredns/coredns/tree/master/plugin/metrics[prometheus exporter endpoint]. + +The default metricset is `stats`. + +[float] +=== Compatibility + +The CoreDNS module is tested with CoreDNS 1.5.0 + + +[float] +=== Dashboard + +The CoreDNS module comes with a predefined dashboard. For example: + +image::./images/metricbeat_coredns_dashboard.png[] + + +[float] +=== Example configuration + +The coredns module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: coredns + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9153"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::coredns/stats.asciidoc[] + diff --git a/metricbeat/docs/modules/coredns/stats.asciidoc b/metricbeat/docs/modules/coredns/stats.asciidoc new file mode 100644 index 000000000000..99669c71cc4e --- /dev/null +++ b/metricbeat/docs/modules/coredns/stats.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-coredns-stats]] +=== coredns stats metricset + +include::../../../../x-pack/metricbeat/module/coredns/stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/coredns/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/couchbase.asciidoc b/metricbeat/docs/modules/couchbase.asciidoc new file mode 100644 index 000000000000..b2520c45f7bd --- /dev/null +++ b/metricbeat/docs/modules/couchbase.asciidoc @@ -0,0 +1,60 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-couchbase]] +== Couchbase module + +This module periodically fetches metrics from https://www.couchbase.com/[Couchbase] +servers. The default metricsets are `bucket`, `cluster`, `node`. + +[float] +=== Compatibility + +The Couchbase module is tested with Couchbase 4.5.1. + + +[float] +=== Dashboard + +The Couchbase module comes with a predefined dashboard for Couchbase cluster, node, bucket specific stats. For example: + +image::./images/metricbeat-couchbase-overview.png[] + + +[float] +=== Example configuration + +The Couchbase module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: couchbase + metricsets: ["bucket", "cluster", "node"] + period: 10s + hosts: ["localhost:8091"] + enabled: true +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::couchbase/bucket.asciidoc[] + +include::couchbase/cluster.asciidoc[] + +include::couchbase/node.asciidoc[] + diff --git a/metricbeat/docs/modules/couchbase/bucket.asciidoc b/metricbeat/docs/modules/couchbase/bucket.asciidoc new file mode 100644 index 000000000000..02e8c67a7361 --- /dev/null +++ b/metricbeat/docs/modules/couchbase/bucket.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-couchbase-bucket]] +=== Couchbase bucket metricset + +include::../../../module/couchbase/bucket/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/couchbase/bucket/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/couchbase/cluster.asciidoc b/metricbeat/docs/modules/couchbase/cluster.asciidoc new file mode 100644 index 000000000000..983fbfecc79e --- /dev/null +++ b/metricbeat/docs/modules/couchbase/cluster.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-couchbase-cluster]] +=== Couchbase cluster metricset + +include::../../../module/couchbase/cluster/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/couchbase/cluster/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/couchbase/node.asciidoc b/metricbeat/docs/modules/couchbase/node.asciidoc new file mode 100644 index 000000000000..a91469d61c04 --- /dev/null +++ b/metricbeat/docs/modules/couchbase/node.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-couchbase-node]] +=== Couchbase node metricset + +include::../../../module/couchbase/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/couchbase/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/couchdb.asciidoc b/metricbeat/docs/modules/couchdb.asciidoc new file mode 100644 index 000000000000..5867e5a02a58 --- /dev/null +++ b/metricbeat/docs/modules/couchdb.asciidoc @@ -0,0 +1,49 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-couchdb]] +== couchdb module + +This is the couchdb module. + +The default metricset is `server`. + +[float] +=== Compatibility + +The Couchdb module is tested with Couchdb 1.7. + + +[float] +=== Dashboard + +The CouchDB module comes with a predefined dashboard for CouchDB database specific stats. For example: + +image::./images/metricbeat-couchdb-overview.png[] + + +[float] +=== Example configuration + +The couchdb module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: couchdb + metricsets: ["server"] + period: 10s + hosts: ["localhost:5984"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::couchdb/server.asciidoc[] + diff --git a/metricbeat/docs/modules/couchdb/server.asciidoc b/metricbeat/docs/modules/couchdb/server.asciidoc new file mode 100644 index 000000000000..edceac1e0e05 --- /dev/null +++ b/metricbeat/docs/modules/couchdb/server.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-couchdb-server]] +=== couchdb server metricset + +include::../../../module/couchdb/server/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/couchdb/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc new file mode 100644 index 000000000000..a9ceb0aa1fb2 --- /dev/null +++ b/metricbeat/docs/modules/docker.asciidoc @@ -0,0 +1,104 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-docker]] +== Docker module + +This module fetches metrics from https://www.docker.com/[Docker] containers. The default metricsets are: `container`, `cpu`, `diskio`, `healthcheck`, `info`, `memory` and `network`. The `image` metricset is not enabled by default. + +[float] +=== Compatibility + +The Docker module is currently tested on Linux and Mac with the community +edition engine, versions 1.11 and 17.09.0-ce. It is not tested on Windows, +but it should also work there. + +[float] +=== Module-specific configuration notes + +It is strongly recommended that you run Docker metricsets with a +<> that is 3 seconds or longer. The request to the +Docker API already takes up to 2 seconds. Specifying less than 3 seconds will +result in requests that timeout, and no data will be reported for those +requests. + + +[float] +=== Example configuration + +The Docker module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "event" + - "healthcheck" + - "info" + #- "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s + enabled: true + + # If set to true, replace dots in labels with `_`. + #labels.dedot: false + + # If set to true, collects metrics per core. + #cpu.cores: true + + # To connect to Docker over TLS you must specify a client and CA certificate. + #ssl: + #certificate_authority: "/etc/pki/root/ca.pem" + #certificate: "/etc/pki/client/cert.pem" + #key: "/etc/pki/client/cert.key" +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +include::docker/container.asciidoc[] + +include::docker/cpu.asciidoc[] + +include::docker/diskio.asciidoc[] + +include::docker/event.asciidoc[] + +include::docker/healthcheck.asciidoc[] + +include::docker/image.asciidoc[] + +include::docker/info.asciidoc[] + +include::docker/memory.asciidoc[] + +include::docker/network.asciidoc[] + diff --git a/metricbeat/docs/modules/docker/container.asciidoc b/metricbeat/docs/modules/docker/container.asciidoc new file mode 100644 index 000000000000..9472efbc1786 --- /dev/null +++ b/metricbeat/docs/modules/docker/container.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-container]] +=== Docker container metricset + +include::../../../module/docker/container/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/container/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/cpu.asciidoc b/metricbeat/docs/modules/docker/cpu.asciidoc new file mode 100644 index 000000000000..5b2d57f52579 --- /dev/null +++ b/metricbeat/docs/modules/docker/cpu.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-cpu]] +=== Docker cpu metricset + +include::../../../module/docker/cpu/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/cpu/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/diskio.asciidoc b/metricbeat/docs/modules/docker/diskio.asciidoc new file mode 100644 index 000000000000..3c5b4dcde684 --- /dev/null +++ b/metricbeat/docs/modules/docker/diskio.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-diskio]] +=== Docker diskio metricset + +include::../../../module/docker/diskio/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/diskio/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/event.asciidoc b/metricbeat/docs/modules/docker/event.asciidoc new file mode 100644 index 000000000000..b9fb636be1bf --- /dev/null +++ b/metricbeat/docs/modules/docker/event.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-event]] +=== Docker event metricset + +include::../../../module/docker/event/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/event/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/healthcheck.asciidoc b/metricbeat/docs/modules/docker/healthcheck.asciidoc new file mode 100644 index 000000000000..90406f556a38 --- /dev/null +++ b/metricbeat/docs/modules/docker/healthcheck.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-healthcheck]] +=== Docker healthcheck metricset + +include::../../../module/docker/healthcheck/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/healthcheck/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/image.asciidoc b/metricbeat/docs/modules/docker/image.asciidoc new file mode 100644 index 000000000000..5983f21b7ed8 --- /dev/null +++ b/metricbeat/docs/modules/docker/image.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-image]] +=== Docker image metricset + +include::../../../module/docker/image/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/image/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/info.asciidoc b/metricbeat/docs/modules/docker/info.asciidoc new file mode 100644 index 000000000000..9766b8caa60b --- /dev/null +++ b/metricbeat/docs/modules/docker/info.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-info]] +=== Docker info metricset + +include::../../../module/docker/info/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/info/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/memory.asciidoc b/metricbeat/docs/modules/docker/memory.asciidoc new file mode 100644 index 000000000000..116a400d365f --- /dev/null +++ b/metricbeat/docs/modules/docker/memory.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-memory]] +=== Docker memory metricset + +include::../../../module/docker/memory/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/docker/network.asciidoc b/metricbeat/docs/modules/docker/network.asciidoc new file mode 100644 index 000000000000..352993c1fd06 --- /dev/null +++ b/metricbeat/docs/modules/docker/network.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-docker-network]] +=== Docker network metricset + +include::../../../module/docker/network/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/network/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/dropwizard.asciidoc b/metricbeat/docs/modules/dropwizard.asciidoc new file mode 100644 index 000000000000..de8d439fde01 --- /dev/null +++ b/metricbeat/docs/modules/dropwizard.asciidoc @@ -0,0 +1,45 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-dropwizard]] +== Dropwizard module + +This is the http://dropwizard.io[Dropwizard] module. The default metricset is `collector`. + +[float] +=== Compatibility + +The Dropwizard module is tested with dropwizard metrics 3.1.0. + + +[float] +=== Example configuration + +The Dropwizard module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: dropwizard + metricsets: ["collector"] + period: 10s + hosts: ["localhost:8080"] + metrics_path: /metrics/metrics + namespace: example + enabled: true +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::dropwizard/collector.asciidoc[] + diff --git a/metricbeat/docs/modules/dropwizard/collector.asciidoc b/metricbeat/docs/modules/dropwizard/collector.asciidoc new file mode 100644 index 000000000000..4b86fd093673 --- /dev/null +++ b/metricbeat/docs/modules/dropwizard/collector.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-dropwizard-collector]] +=== Dropwizard collector metricset + +include::../../../module/dropwizard/collector/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/dropwizard/collector/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc new file mode 100644 index 000000000000..484519f7426d --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -0,0 +1,108 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-elasticsearch]] +== Elasticsearch module + +There are two modules that collect metrics about {es}: + +* The Elasticsearch module contains a minimal set of metrics to enable +monitoring of Elasticsearch across multiple versions. The default metricsets in +this module are `node` and `node_stats`. +* The Elasticsearch X-Pack module enables you to monitor more Elasticsearch +metrics with our {stack} {monitor-features}. The default metricsets in this +module are `ccr`, `cluster_stats`, `index`, `index_recovery`, `index_summary`, +`ml_job`, `node_stats`, and `shard`. + +[float] +=== Compatibility + +The Elasticsearch module is tested with Elasticsearch 6.3 and is expected to +work with all 6.x versions. + + +[float] +=== Example configuration + +The Elasticsearch module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: elasticsearch + metricsets: + - node + - node_stats + #- index + #- index_recovery + #- index_summary + #- shard + #- ml_job + period: 10s + hosts: ["http://localhost:9200"] + #username: "elastic" + #password: "changeme" + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to false to fetch all entries + #index_recovery.active_only: true + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +include::elasticsearch/ccr.asciidoc[] + +include::elasticsearch/cluster_stats.asciidoc[] + +include::elasticsearch/enrich.asciidoc[] + +include::elasticsearch/index.asciidoc[] + +include::elasticsearch/index_recovery.asciidoc[] + +include::elasticsearch/index_summary.asciidoc[] + +include::elasticsearch/ml_job.asciidoc[] + +include::elasticsearch/node.asciidoc[] + +include::elasticsearch/node_stats.asciidoc[] + +include::elasticsearch/pending_tasks.asciidoc[] + +include::elasticsearch/shard.asciidoc[] + diff --git a/metricbeat/docs/modules/elasticsearch/ccr.asciidoc b/metricbeat/docs/modules/elasticsearch/ccr.asciidoc new file mode 100644 index 000000000000..467d1de0bf3d --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/ccr.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-ccr]] +=== Elasticsearch ccr metricset + +include::../../../module/elasticsearch/ccr/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/ccr/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/cluster_stats.asciidoc b/metricbeat/docs/modules/elasticsearch/cluster_stats.asciidoc new file mode 100644 index 000000000000..9c4a99356a09 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/cluster_stats.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-cluster_stats]] +=== Elasticsearch cluster_stats metricset + +include::../../../module/elasticsearch/cluster_stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/cluster_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/enrich.asciidoc b/metricbeat/docs/modules/elasticsearch/enrich.asciidoc new file mode 100644 index 000000000000..9e4f56ba18b1 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/enrich.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-enrich]] +=== Elasticsearch enrich metricset + +include::../../../module/elasticsearch/enrich/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/enrich/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/index.asciidoc b/metricbeat/docs/modules/elasticsearch/index.asciidoc new file mode 100644 index 000000000000..933a5e788b4d --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/index.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-index]] +=== Elasticsearch index metricset + +include::../../../module/elasticsearch/index/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/index/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/index_recovery.asciidoc b/metricbeat/docs/modules/elasticsearch/index_recovery.asciidoc new file mode 100644 index 000000000000..cef143017b5c --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/index_recovery.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-index_recovery]] +=== Elasticsearch index_recovery metricset + +include::../../../module/elasticsearch/index_recovery/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/index_recovery/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/index_summary.asciidoc b/metricbeat/docs/modules/elasticsearch/index_summary.asciidoc new file mode 100644 index 000000000000..b74c5c626812 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/index_summary.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-index_summary]] +=== Elasticsearch index_summary metricset + +include::../../../module/elasticsearch/index_summary/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/index_summary/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/ml_job.asciidoc b/metricbeat/docs/modules/elasticsearch/ml_job.asciidoc new file mode 100644 index 000000000000..ffbd9b9bac3b --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/ml_job.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-ml_job]] +=== Elasticsearch ml_job metricset + +include::../../../module/elasticsearch/ml_job/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/ml_job/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/node.asciidoc b/metricbeat/docs/modules/elasticsearch/node.asciidoc new file mode 100644 index 000000000000..3e6dfa372179 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/node.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-node]] +=== Elasticsearch node metricset + +include::../../../module/elasticsearch/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/node_stats.asciidoc b/metricbeat/docs/modules/elasticsearch/node_stats.asciidoc new file mode 100644 index 000000000000..4b8dd9339912 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/node_stats.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-node_stats]] +=== Elasticsearch node_stats metricset + +include::../../../module/elasticsearch/node_stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/node_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/pending_tasks.asciidoc b/metricbeat/docs/modules/elasticsearch/pending_tasks.asciidoc new file mode 100644 index 000000000000..b278af1944e2 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/pending_tasks.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-pending_tasks]] +=== Elasticsearch pending_tasks metricset + +include::../../../module/elasticsearch/pending_tasks/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/pending_tasks/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/elasticsearch/shard.asciidoc b/metricbeat/docs/modules/elasticsearch/shard.asciidoc new file mode 100644 index 000000000000..b3cf2f3cdd36 --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/shard.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-elasticsearch-shard]] +=== Elasticsearch shard metricset + +include::../../../module/elasticsearch/shard/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/shard/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/envoyproxy.asciidoc b/metricbeat/docs/modules/envoyproxy.asciidoc new file mode 100644 index 000000000000..bc7d1c341d7a --- /dev/null +++ b/metricbeat/docs/modules/envoyproxy.asciidoc @@ -0,0 +1,41 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-envoyproxy]] +== envoyproxy module + +This is the envoyproxy module. + +The default metricset is `server`. + +[float] +=== Compatibility + +The envoyproxy module is tested with Envoy 1.7.0 and 1.12.0. + + +[float] +=== Example configuration + +The envoyproxy module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: envoyproxy + metricsets: ["server"] + period: 10s + hosts: ["localhost:9901"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::envoyproxy/server.asciidoc[] + diff --git a/metricbeat/docs/modules/envoyproxy/server.asciidoc b/metricbeat/docs/modules/envoyproxy/server.asciidoc new file mode 100644 index 000000000000..609ca4204b36 --- /dev/null +++ b/metricbeat/docs/modules/envoyproxy/server.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-envoyproxy-server]] +=== envoyproxy server metricset + +include::../../../module/envoyproxy/server/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/envoyproxy/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/etcd.asciidoc b/metricbeat/docs/modules/etcd.asciidoc new file mode 100644 index 000000000000..0e12f80d0427 --- /dev/null +++ b/metricbeat/docs/modules/etcd.asciidoc @@ -0,0 +1,60 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-etcd]] +== Etcd module + +This module targets Etcd V2 and V3. + +When using V2, metrics are collected using https://coreos.com/etcd/docs/latest/v2/api.html[Etcd v2 API]. +When using V3, metrics are retrieved from the `/metrics` endpoint as intended for https://coreos.com/etcd/docs/latest/metrics.html[Etcd v3] + +When using V3, metricsest are bundled into `metrics` +When using V2, metricsets available are `leader`, `self` and `store`. + +[float] +=== Compatibility + +The etcd module is tested with etcd 3.2 and 3.3. + + +[float] +=== Example configuration + +The Etcd module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: etcd + metricsets: ["leader", "self", "store"] + period: 10s + hosts: ["localhost:2379"] +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::etcd/leader.asciidoc[] + +include::etcd/metrics.asciidoc[] + +include::etcd/self.asciidoc[] + +include::etcd/store.asciidoc[] + diff --git a/metricbeat/docs/modules/etcd/leader.asciidoc b/metricbeat/docs/modules/etcd/leader.asciidoc new file mode 100644 index 000000000000..368acfebe3d8 --- /dev/null +++ b/metricbeat/docs/modules/etcd/leader.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-etcd-leader]] +=== Etcd leader metricset + +include::../../../module/etcd/leader/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/etcd/leader/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/etcd/metrics.asciidoc b/metricbeat/docs/modules/etcd/metrics.asciidoc new file mode 100644 index 000000000000..6936b16f4eb7 --- /dev/null +++ b/metricbeat/docs/modules/etcd/metrics.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-etcd-metrics]] +=== Etcd metrics metricset + +beta[] + +include::../../../module/etcd/metrics/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/etcd/metrics/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/etcd/self.asciidoc b/metricbeat/docs/modules/etcd/self.asciidoc new file mode 100644 index 000000000000..21888d5d8e67 --- /dev/null +++ b/metricbeat/docs/modules/etcd/self.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-etcd-self]] +=== Etcd self metricset + +include::../../../module/etcd/self/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/etcd/self/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/etcd/store.asciidoc b/metricbeat/docs/modules/etcd/store.asciidoc new file mode 100644 index 000000000000..898c7ad42933 --- /dev/null +++ b/metricbeat/docs/modules/etcd/store.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-etcd-store]] +=== Etcd store metricset + +include::../../../module/etcd/store/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/etcd/store/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/golang.asciidoc b/metricbeat/docs/modules/golang.asciidoc new file mode 100644 index 000000000000..9338042a8d37 --- /dev/null +++ b/metricbeat/docs/modules/golang.asciidoc @@ -0,0 +1,47 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-golang]] +== Golang module + +The golang module collects metrics by submitting HTTP GET requests to https://golang.org/pkg/expvar/[golang-expvar-api]. + + +[float] +=== Example configuration + +The Golang module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: golang + #metricsets: + # - expvar + # - heap + period: 10s + hosts: ["localhost:6060"] + heap.path: "/debug/vars" + expvar: + namespace: "example" + path: "/debug/vars" +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::golang/expvar.asciidoc[] + +include::golang/heap.asciidoc[] + diff --git a/metricbeat/docs/modules/golang/expvar.asciidoc b/metricbeat/docs/modules/golang/expvar.asciidoc new file mode 100644 index 000000000000..c2a81c6e4c9f --- /dev/null +++ b/metricbeat/docs/modules/golang/expvar.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-golang-expvar]] +=== Golang expvar metricset + +include::../../../module/golang/expvar/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/golang/expvar/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/golang/heap.asciidoc b/metricbeat/docs/modules/golang/heap.asciidoc new file mode 100644 index 000000000000..8629a71353a0 --- /dev/null +++ b/metricbeat/docs/modules/golang/heap.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-golang-heap]] +=== Golang heap metricset + +include::../../../module/golang/heap/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/golang/heap/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/graphite.asciidoc b/metricbeat/docs/modules/graphite.asciidoc new file mode 100644 index 000000000000..e0e4821cb37b --- /dev/null +++ b/metricbeat/docs/modules/graphite.asciidoc @@ -0,0 +1,54 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-graphite]] +== Graphite module + +This is the Graphite module. + +The default metricset is `server`. + + +[float] +=== Example configuration + +The Graphite module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: graphite + metricsets: ["server"] + enabled: true + + # Host address to listen on. Default localhost. + #host: localhost + + # Listening port. Default 2003. + #port: 2003 + + # Protocol to listen on. This can be udp or tcp. Default udp. + #protocol: "udp" + + # Receive buffer size in bytes + #receive_buffer_size: 1024 + + #templates: + # - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats + # namespace: "test" + # template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash + # delimiter: "_" + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::graphite/server.asciidoc[] + diff --git a/metricbeat/docs/modules/graphite/server.asciidoc b/metricbeat/docs/modules/graphite/server.asciidoc new file mode 100644 index 000000000000..0cc8bfb3c5dc --- /dev/null +++ b/metricbeat/docs/modules/graphite/server.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-graphite-server]] +=== Graphite server metricset + +include::../../../module/graphite/server/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/graphite/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/haproxy.asciidoc b/metricbeat/docs/modules/haproxy.asciidoc new file mode 100644 index 000000000000..588b996e1e79 --- /dev/null +++ b/metricbeat/docs/modules/haproxy.asciidoc @@ -0,0 +1,77 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-haproxy]] +== HAProxy module + +This module collects stats from http://www.haproxy.org/[HAProxy]. It supports +collection from using TCP sockets or HTTP with or without basic authentication. + +To configure HAProxy to collect stats, you must enable the stats module, it can +be done by enabling a TCP socket, or by adding an HTTP stats frontend. + +Metricbeat can collect two metric sets from HAProxy, `info` and `stats`. `info` +is not available when using HTTP stats frontend. + +For example, to enable stats reporting via any local IP on port 14567, place +this statement under the `global` or `default` section of the haproxy config: + +[source,haproxy] +---- + stats socket 127.0.0.1:14567 +---- + +NOTE: You should use an internal private IP, or secure this with a firewall +rule, so that only designated hosts can access this data. + +To configure the HTTP stats frontend, a frontend with stats enabled has to +be added. For example, to open this frontend to any IP on port 14567 with +required authentication add this to the haproxy config: + +[source,haproxy] +---- + listen stats + bind 0.0.0.0:14567 + stats enable + stats uri /stats + stats auth admin:admin +---- + +The default metricsets are `info`and `stat`. + +[float] +=== Compatibility + +The HAProxy metricsets are tested with HAProxy versions from 1.6, 1.7 to 1.8. + + +[float] +=== Example configuration + +The HAProxy module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: haproxy + metricsets: ["info", "stat"] + period: 10s + hosts: ["tcp://127.0.0.1:14567"] + enabled: true +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::haproxy/info.asciidoc[] + +include::haproxy/stat.asciidoc[] + diff --git a/metricbeat/docs/modules/haproxy/info.asciidoc b/metricbeat/docs/modules/haproxy/info.asciidoc new file mode 100644 index 000000000000..06c62a015393 --- /dev/null +++ b/metricbeat/docs/modules/haproxy/info.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-haproxy-info]] +=== HAProxy info metricset + +include::../../../module/haproxy/info/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/haproxy/info/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/haproxy/stat.asciidoc b/metricbeat/docs/modules/haproxy/stat.asciidoc new file mode 100644 index 000000000000..9dbeb62776e4 --- /dev/null +++ b/metricbeat/docs/modules/haproxy/stat.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-haproxy-stat]] +=== HAProxy stat metricset + +include::../../../module/haproxy/stat/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/haproxy/stat/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc new file mode 100644 index 000000000000..326873a37f16 --- /dev/null +++ b/metricbeat/docs/modules/http.asciidoc @@ -0,0 +1,72 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-http]] +== HTTP module + +The HTTP module is a Metricbeat module used to call arbitrary HTTP endpoints for which a dedicated Metricbeat module is not available. + +Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. It is recommended to install a Metricbeat instance on each host from which data should be fetched. + +This module is inspired by the Logstash https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html[http_poller] input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured output channels, e.g. Logstash or Elasticsearch. + +This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. + +NOTE: As the HTTP metricsets also fetch headers, this can lead to lots of fields in Elasticsearch in case there are many different headers. If this is the case for you and you don't need the headers, we recommend to use processors to filter out the header field. + + +[float] +=== Example configuration + +The HTTP module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: http + #metricsets: + # - json + period: 10s + hosts: ["localhost:80"] + namespace: "json_namespace" + path: "/" + #body: "" + #method: "GET" + #username: "user" + #password: "secret" + #request.enabled: false + #response.enabled: false + #json.is_array: false + #dedot.enabled: false + +- module: http + #metricsets: + # - server + host: "localhost" + port: "8080" + enabled: false + #paths: + # - path: "/foo" + # namespace: "foo" + # fields: # added to the the response in root. overwrites existing fields + # key: "value" +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::http/json.asciidoc[] + +include::http/server.asciidoc[] + diff --git a/metricbeat/docs/modules/http/json.asciidoc b/metricbeat/docs/modules/http/json.asciidoc new file mode 100644 index 000000000000..2fa2e48c2d3e --- /dev/null +++ b/metricbeat/docs/modules/http/json.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-http-json]] +=== HTTP json metricset + +include::../../../module/http/json/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/http/json/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/http/server.asciidoc b/metricbeat/docs/modules/http/server.asciidoc new file mode 100644 index 000000000000..bf5437ab7c09 --- /dev/null +++ b/metricbeat/docs/modules/http/server.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-http-server]] +=== HTTP server metricset + +include::../../../module/http/server/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/http/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc new file mode 100644 index 000000000000..78fad39cb021 --- /dev/null +++ b/metricbeat/docs/modules/iis.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-iis]] +== iis module + +beta[] + +This is the iis module. + + + +[float] +=== Example configuration + +The iis module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: iis + metricsets: + - webserver + enabled: true + period: 10s + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::iis/webserver.asciidoc[] + +include::iis/website.asciidoc[] + diff --git a/metricbeat/docs/modules/iis/webserver.asciidoc b/metricbeat/docs/modules/iis/webserver.asciidoc new file mode 100644 index 000000000000..4352ab768f01 --- /dev/null +++ b/metricbeat/docs/modules/iis/webserver.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-iis-webserver]] +=== iis webserver metricset + +beta[] + +include::../../../module/iis/webserver/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/webserver/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/iis/website.asciidoc b/metricbeat/docs/modules/iis/website.asciidoc new file mode 100644 index 000000000000..1132d876ad49 --- /dev/null +++ b/metricbeat/docs/modules/iis/website.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-iis-website]] +=== iis website metricset + +beta[] + +include::../../../module/iis/website/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/website/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc new file mode 100644 index 000000000000..fa91aedeaf83 --- /dev/null +++ b/metricbeat/docs/modules/jolokia.asciidoc @@ -0,0 +1,74 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-jolokia]] +== Jolokia module + +This module collects metrics from +https://jolokia.org/reference/html/agents.html[Jolokia agents] running on a +target JMX server or dedicated proxy server. The default metricset is `jmx`. + +To collect metrics, {beatname_uc} communicates with a Jolokia HTTP/REST +endpoint that exposes the JMX metrics over HTTP/REST/JSON. + +[float] +=== Compatibility + +The Jolokia module is tested with Jolokia 1.5.0. It should work with version +1.2.2 and later. + + + +[float] +=== Example configuration + +The Jolokia module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: jolokia + #metricsets: ["jmx"] + period: 10s + hosts: ["localhost"] + namespace: "metrics" + #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + #username: "user" + #password: "secret" + jmx.mappings: + #- mbean: 'java.lang:type=Runtime' + # attributes: + # - attr: Uptime + # field: uptime + #- mbean: 'java.lang:type=Memory' + # attributes: + # - attr: HeapMemoryUsage + # field: memory.heap_usage + # - attr: NonHeapMemoryUsage + # field: memory.non_heap_usage + # GC Metrics - this depends on what is available on your JVM + #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' + # attributes: + # - attr: CollectionTime + # field: gc.cms_collection_time + # - attr: CollectionCount + # field: gc.cms_collection_count + + jmx.application: + jmx.instance: +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::jolokia/jmx.asciidoc[] + diff --git a/metricbeat/docs/modules/jolokia/jmx.asciidoc b/metricbeat/docs/modules/jolokia/jmx.asciidoc new file mode 100644 index 000000000000..20f17f58d92e --- /dev/null +++ b/metricbeat/docs/modules/jolokia/jmx.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-jolokia-jmx]] +=== Jolokia jmx metricset + +include::../../../module/jolokia/jmx/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/jolokia/jmx/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc new file mode 100644 index 000000000000..72cc0a0fffa7 --- /dev/null +++ b/metricbeat/docs/modules/kafka.asciidoc @@ -0,0 +1,131 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-kafka]] +== Kafka module + +This is the Kafka module. + +The default metricsets are `consumergroup` and `partition`. + +If authorization is configured in the Kafka cluster, the following ACLs are +required for the Metricbeat user: + +* READ Topic, for the topics to be monitored +* DESCRIBE Group, for the groups to be monitored + +For example, if the `stats` user is being used for Metricbeat, to monitor all +topics and all consumer groups, ACLS can be granted with the following commands: + +[source,shell] +----- +kafka-acls --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:stats --operation Read --topic '*' +kafka-acls --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:stats --operation Describe --group '*' +----- + +[float] +=== Compatibility + +This module is tested with Kafka 0.10.2.1, 1.1.0 and 2.1.1. + +The Broker, Producer, Consumer metricsets require <>to fetch JMX metrics. Refer to the link for Jolokia's compatibility notes. + +[float] +=== Usage +The Broker, Producer, Consumer metricsets require <>to fetch JMX metrics. Refer to those Metricsets' documentation about how to use Jolokia. + + +[float] +=== Dashboard + +The Kafka module comes with a predefined dashboard. For example: + +image::./images/metricbeat_kafka_dashboard.png[] + + +[float] +=== Example configuration + +The Kafka module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +# Kafka metrics collected using the Kafka protocol +- module: kafka + #metricsets: + # - partition + # - consumergroup + period: 10s + hosts: ["localhost:9092"] + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" + +# Metrics collected from a Kafka broker using Jolokia +#- module: kafka +# metricsets: +# - broker +# period: 10s +# hosts: ["localhost:8779"] + +# Metrics collected from a Java Kafka consumer using Jolokia +#- module: kafka +# metricsets: +# - consumer +# period: 10s +# hosts: ["localhost:8774"] + +# Metrics collected from a Java Kafka producer using Jolokia +#- module: kafka +# metricsets: +# - producer +# period: 10s +# hosts: ["localhost:8775"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +include::kafka/broker.asciidoc[] + +include::kafka/consumer.asciidoc[] + +include::kafka/consumergroup.asciidoc[] + +include::kafka/partition.asciidoc[] + +include::kafka/producer.asciidoc[] + diff --git a/metricbeat/docs/modules/kafka/broker.asciidoc b/metricbeat/docs/modules/kafka/broker.asciidoc new file mode 100644 index 000000000000..9ef1c702ce2a --- /dev/null +++ b/metricbeat/docs/modules/kafka/broker.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-broker]] +=== Kafka broker metricset + +beta[] + +include::../../../module/kafka/broker/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/broker/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka/consumer.asciidoc b/metricbeat/docs/modules/kafka/consumer.asciidoc new file mode 100644 index 000000000000..a30bd9001188 --- /dev/null +++ b/metricbeat/docs/modules/kafka/consumer.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-consumer]] +=== Kafka consumer metricset + +beta[] + +include::../../../module/kafka/consumer/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/consumer/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka/consumergroup.asciidoc b/metricbeat/docs/modules/kafka/consumergroup.asciidoc new file mode 100644 index 000000000000..10939b7743c2 --- /dev/null +++ b/metricbeat/docs/modules/kafka/consumergroup.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-consumergroup]] +=== Kafka consumergroup metricset + +include::../../../module/kafka/consumergroup/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/consumergroup/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka/partition.asciidoc b/metricbeat/docs/modules/kafka/partition.asciidoc new file mode 100644 index 000000000000..7fd178edcdf2 --- /dev/null +++ b/metricbeat/docs/modules/kafka/partition.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-partition]] +=== Kafka partition metricset + +include::../../../module/kafka/partition/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/partition/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kafka/producer.asciidoc b/metricbeat/docs/modules/kafka/producer.asciidoc new file mode 100644 index 000000000000..d7e9e6b9d536 --- /dev/null +++ b/metricbeat/docs/modules/kafka/producer.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kafka-producer]] +=== Kafka producer metricset + +beta[] + +include::../../../module/kafka/producer/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kafka/producer/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc new file mode 100644 index 000000000000..a878e843071f --- /dev/null +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -0,0 +1,57 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-kibana]] +== Kibana module + +There are two modules that collect metrics about {kib}: + +* The Kibana module tracks only the high-level metrics. The default metricset in +this module is `status`. +* The Kibana X-Pack module enables you to monitor more Kibana metrics with our +{stack} {monitor-features}. The default metricset in this module is `stats`. + +[float] +=== Compatibility + +The Kibana module is tested with Kibana 6.3. + + +[float] +=== Example configuration + +The Kibana module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: kibana + metricsets: ["status"] + period: 10s + hosts: ["localhost:5601"] + basepath: "" + enabled: true + + # Set to true to send data collected by module to X-Pack + # Monitoring instead of metricbeat-* indices. + #xpack.enabled: false +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::kibana/stats.asciidoc[] + +include::kibana/status.asciidoc[] + diff --git a/metricbeat/docs/modules/kibana/stats.asciidoc b/metricbeat/docs/modules/kibana/stats.asciidoc new file mode 100644 index 000000000000..b4ad84d2777d --- /dev/null +++ b/metricbeat/docs/modules/kibana/stats.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kibana-stats]] +=== Kibana stats metricset + +include::../../../module/kibana/stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kibana/status.asciidoc b/metricbeat/docs/modules/kibana/status.asciidoc new file mode 100644 index 000000000000..c1b6fc139876 --- /dev/null +++ b/metricbeat/docs/modules/kibana/status.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kibana-status]] +=== Kibana status metricset + +include::../../../module/kibana/status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc new file mode 100644 index 000000000000..9c5b27279232 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -0,0 +1,235 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-kubernetes]] +== Kubernetes module + +This module fetches metrics from Kubernetes several components: + +- https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/[kubelet] +- https://github.com/kubernetes/kube-state-metrics[kube-state-metrics] +- https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/[apiserver] +- https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/[controller-manager] +- https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/[scheduler] +- https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/[proxy] + +All metricsets with the `state_` prefix require `hosts` field pointing to kube-state-metrics +service within the cluster, while the rest should be pointed to kubelet service. Check the +example configuration on how to do it. + +The default metricsets are `container`, `node`, `pod`, `system` and `volume`. + +[float] +=== Compatibility + +The Kubernetes module is tested with Kubernetes 1.13.x and 1.14.x + +[float] +=== Dashboard + +Kubernetes module is shipped including default dashboards for `apiserver`, `controllermanager`, `scheduler` and `proxy`. + +If you are using HA for those components, be aware that when gathering data from all instances the dashboard will usually show and average of the metrics. For those scenarios filtering by hosts or service address is possible. + +Dashboards for `controllermanager` `scheduler` and `proxy` are not compatible with kibana versions below `7.2.0` + +Kubernetes controller manager example: + +image::./images/metricbeat-kubernetes-controllermanager.png[] + + +Kubernetes scheduler example: + +image::./images/metricbeat_kubernetes_scheduler.png[] + + +Kubernetes proxy example: + +image::./images/metricbeat-kubernetes-proxy.png[] + + +[float] +=== Example configuration + +The Kubernetes module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +# Node metrics, from kubelet: +- module: kubernetes + metricsets: + - container + - node + - pod + - system + - volume + period: 10s + hosts: ["localhost:10255"] + enabled: true + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + #ssl.certificate: "/etc/pki/client/cert.pem" + #ssl.key: "/etc/pki/client/cert.key" + + # Enriching parameters: + add_metadata: true + # When used outside the cluster: + #host: node_name + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + +# State metrics from kube-state-metrics service: +- module: kubernetes + enabled: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_statefulset + - state_pod + - state_container + - state_cronjob + - state_resourcequota + - state_service + period: 10s + hosts: ["kube-state-metrics:8080"] + + # Enriching parameters: + add_metadata: true + # When used outside the cluster: + #host: node_name + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + +# Kubernetes events +- module: kubernetes + enabled: true + metricsets: + - event + +# Kubernetes API server +# (when running metricbeat as a pod) +- module: kubernetes + enabled: true + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + +# Kubernetes proxy server +# (when running metricbeat locally at hosts or as a daemonset + host network) +- module: kubernetes + enabled: true + metricsets: + - proxy + hosts: ["localhost:10249"] + period: 10s + +# Kubernetes controller manager +# (URL and deployment will need to be modified to match the controller manager deployment) +- module: kubernetes + enabled: true + metricsets: + - controllermanager + hosts: ["http://localhost:10252"] + period: 10s + +# Kubernetes scheduler +# (URL and deployment method will need to be adapted to match scheduler deployment) +- module: kubernetes + enabled: true + metricsets: + - scheduler + hosts: ["localhost:10251"] + period: 10s +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +include::kubernetes/apiserver.asciidoc[] + +include::kubernetes/container.asciidoc[] + +include::kubernetes/controllermanager.asciidoc[] + +include::kubernetes/event.asciidoc[] + +include::kubernetes/node.asciidoc[] + +include::kubernetes/pod.asciidoc[] + +include::kubernetes/proxy.asciidoc[] + +include::kubernetes/scheduler.asciidoc[] + +include::kubernetes/state_container.asciidoc[] + +include::kubernetes/state_cronjob.asciidoc[] + +include::kubernetes/state_deployment.asciidoc[] + +include::kubernetes/state_node.asciidoc[] + +include::kubernetes/state_pod.asciidoc[] + +include::kubernetes/state_replicaset.asciidoc[] + +include::kubernetes/state_resourcequota.asciidoc[] + +include::kubernetes/state_service.asciidoc[] + +include::kubernetes/state_statefulset.asciidoc[] + +include::kubernetes/system.asciidoc[] + +include::kubernetes/volume.asciidoc[] + diff --git a/metricbeat/docs/modules/kubernetes/apiserver.asciidoc b/metricbeat/docs/modules/kubernetes/apiserver.asciidoc new file mode 100644 index 000000000000..389fec38cbcf --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/apiserver.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-apiserver]] +=== Kubernetes apiserver metricset + +include::../../../module/kubernetes/apiserver/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/apiserver/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/container.asciidoc b/metricbeat/docs/modules/kubernetes/container.asciidoc new file mode 100644 index 000000000000..4028e84306ae --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/container.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-container]] +=== Kubernetes container metricset + +include::../../../module/kubernetes/container/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/container/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/controllermanager.asciidoc b/metricbeat/docs/modules/kubernetes/controllermanager.asciidoc new file mode 100644 index 000000000000..acc08e046303 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/controllermanager.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-controllermanager]] +=== Kubernetes controllermanager metricset + +include::../../../module/kubernetes/controllermanager/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/controllermanager/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/event.asciidoc b/metricbeat/docs/modules/kubernetes/event.asciidoc new file mode 100644 index 000000000000..791958503236 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/event.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-event]] +=== Kubernetes event metricset + +include::../../../module/kubernetes/event/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/event/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/node.asciidoc b/metricbeat/docs/modules/kubernetes/node.asciidoc new file mode 100644 index 000000000000..fdc5a85b9c0f --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/node.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-node]] +=== Kubernetes node metricset + +include::../../../module/kubernetes/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/pod.asciidoc b/metricbeat/docs/modules/kubernetes/pod.asciidoc new file mode 100644 index 000000000000..5fd43885f781 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/pod.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-pod]] +=== Kubernetes pod metricset + +include::../../../module/kubernetes/pod/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/pod/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/proxy.asciidoc b/metricbeat/docs/modules/kubernetes/proxy.asciidoc new file mode 100644 index 000000000000..ed3176735bfe --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/proxy.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-proxy]] +=== Kubernetes proxy metricset + +include::../../../module/kubernetes/proxy/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/proxy/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/scheduler.asciidoc b/metricbeat/docs/modules/kubernetes/scheduler.asciidoc new file mode 100644 index 000000000000..364bc67d6f31 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/scheduler.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-scheduler]] +=== Kubernetes scheduler metricset + +include::../../../module/kubernetes/scheduler/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/scheduler/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_container.asciidoc b/metricbeat/docs/modules/kubernetes/state_container.asciidoc new file mode 100644 index 000000000000..7ed48032c933 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_container.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_container]] +=== Kubernetes state_container metricset + +include::../../../module/kubernetes/state_container/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_container/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_cronjob.asciidoc b/metricbeat/docs/modules/kubernetes/state_cronjob.asciidoc new file mode 100644 index 000000000000..63252dbcf2ae --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_cronjob.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_cronjob]] +=== Kubernetes state_cronjob metricset + +include::../../../module/kubernetes/state_cronjob/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_cronjob/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_deployment.asciidoc b/metricbeat/docs/modules/kubernetes/state_deployment.asciidoc new file mode 100644 index 000000000000..cfc0c833c662 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_deployment.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_deployment]] +=== Kubernetes state_deployment metricset + +include::../../../module/kubernetes/state_deployment/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_deployment/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_node.asciidoc b/metricbeat/docs/modules/kubernetes/state_node.asciidoc new file mode 100644 index 000000000000..517b64513572 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_node.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_node]] +=== Kubernetes state_node metricset + +include::../../../module/kubernetes/state_node/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_pod.asciidoc b/metricbeat/docs/modules/kubernetes/state_pod.asciidoc new file mode 100644 index 000000000000..b56c37c4a0e8 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_pod.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_pod]] +=== Kubernetes state_pod metricset + +include::../../../module/kubernetes/state_pod/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_pod/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_replicaset.asciidoc b/metricbeat/docs/modules/kubernetes/state_replicaset.asciidoc new file mode 100644 index 000000000000..1cced9d3b355 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_replicaset.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_replicaset]] +=== Kubernetes state_replicaset metricset + +include::../../../module/kubernetes/state_replicaset/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_replicaset/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_resourcequota.asciidoc b/metricbeat/docs/modules/kubernetes/state_resourcequota.asciidoc new file mode 100644 index 000000000000..d220ffa99f94 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_resourcequota.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_resourcequota]] +=== Kubernetes state_resourcequota metricset + +include::../../../module/kubernetes/state_resourcequota/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_resourcequota/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_service.asciidoc b/metricbeat/docs/modules/kubernetes/state_service.asciidoc new file mode 100644 index 000000000000..5921d5939718 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_service.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_service]] +=== Kubernetes state_service metricset + +experimental[] + +include::../../../module/kubernetes/state_service/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_service/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/state_statefulset.asciidoc b/metricbeat/docs/modules/kubernetes/state_statefulset.asciidoc new file mode 100644 index 000000000000..e39f7bfb5643 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_statefulset.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_statefulset]] +=== Kubernetes state_statefulset metricset + +include::../../../module/kubernetes/state_statefulset/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_statefulset/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/system.asciidoc b/metricbeat/docs/modules/kubernetes/system.asciidoc new file mode 100644 index 000000000000..512f1675331a --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/system.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-system]] +=== Kubernetes system metricset + +include::../../../module/kubernetes/system/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/system/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kubernetes/volume.asciidoc b/metricbeat/docs/modules/kubernetes/volume.asciidoc new file mode 100644 index 000000000000..63f576c3b657 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/volume.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-volume]] +=== Kubernetes volume metricset + +include::../../../module/kubernetes/volume/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/volume/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/kvm.asciidoc b/metricbeat/docs/modules/kvm.asciidoc new file mode 100644 index 000000000000..f8373184ba15 --- /dev/null +++ b/metricbeat/docs/modules/kvm.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-kvm]] +== kvm module + +beta[] + +This is the kvm module. + + +[float] +=== Example configuration + +The kvm module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: kvm + metricsets: ["dommemstat"] + enabled: true + period: 10s + hosts: ["unix:///var/run/libvirt/libvirt-sock"] + # For remote hosts, setup network access in libvirtd.conf + # and use the tcp scheme: + # hosts: [ "tcp://:16509" ] + + # Timeout to connect to Libvirt server + #timeout: 1s +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::kvm/dommemstat.asciidoc[] + diff --git a/metricbeat/docs/modules/kvm/dommemstat.asciidoc b/metricbeat/docs/modules/kvm/dommemstat.asciidoc new file mode 100644 index 000000000000..c7cc150b9c0f --- /dev/null +++ b/metricbeat/docs/modules/kvm/dommemstat.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kvm-dommemstat]] +=== kvm dommemstat metricset + +beta[] + +include::../../../module/kvm/dommemstat/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kvm/dommemstat/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/logstash.asciidoc b/metricbeat/docs/modules/logstash.asciidoc new file mode 100644 index 000000000000..c29633ce907c --- /dev/null +++ b/metricbeat/docs/modules/logstash.asciidoc @@ -0,0 +1,49 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-logstash]] +== Logstash module + +This is the Logstash module. + +The default metricsets are `node` and `node_stats`. + +[float] +=== Compatibility + +The logstash module is tested with logstash 6.3. + + +[float] +=== Example configuration + +The Logstash module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: logstash + metricsets: ["node", "node_stats"] + enabled: true + period: 10s + hosts: ["localhost:9600"] +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::logstash/node.asciidoc[] + +include::logstash/node_stats.asciidoc[] + diff --git a/metricbeat/docs/modules/logstash/node.asciidoc b/metricbeat/docs/modules/logstash/node.asciidoc new file mode 100644 index 000000000000..19c839943105 --- /dev/null +++ b/metricbeat/docs/modules/logstash/node.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-logstash-node]] +=== Logstash node metricset + +include::../../../module/logstash/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/logstash/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/logstash/node_stats.asciidoc b/metricbeat/docs/modules/logstash/node_stats.asciidoc new file mode 100644 index 000000000000..2a9495faacb4 --- /dev/null +++ b/metricbeat/docs/modules/logstash/node_stats.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-logstash-node_stats]] +=== Logstash node_stats metricset + +include::../../../module/logstash/node_stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/logstash/node_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/memcached.asciidoc b/metricbeat/docs/modules/memcached.asciidoc new file mode 100644 index 000000000000..ca34cdff5c9a --- /dev/null +++ b/metricbeat/docs/modules/memcached.asciidoc @@ -0,0 +1,42 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-memcached]] +== Memcached module + +This is the Memcached module. These metricsets were tested with Memcached version 1.4.35. + +The default metricset is `stats`. + +[float] +=== Compatibility + +The memcached module is tested with memcached 1.4.35. + + +[float] +=== Example configuration + +The Memcached module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: memcached + metricsets: ["stats"] + period: 10s + hosts: ["localhost:11211"] + enabled: true +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::memcached/stats.asciidoc[] + diff --git a/metricbeat/docs/modules/memcached/stats.asciidoc b/metricbeat/docs/modules/memcached/stats.asciidoc new file mode 100644 index 000000000000..f44b8b22991c --- /dev/null +++ b/metricbeat/docs/modules/memcached/stats.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-memcached-stats]] +=== Memcached stats metricset + +include::../../../module/memcached/stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/memcached/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mongodb.asciidoc b/metricbeat/docs/modules/mongodb.asciidoc new file mode 100644 index 000000000000..c7e6e9060fe5 --- /dev/null +++ b/metricbeat/docs/modules/mongodb.asciidoc @@ -0,0 +1,155 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-mongodb]] +== MongoDB module + +This module periodically fetches metrics from https://www.mongodb.com[MongoDB] +servers. + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use MongoDB URLs of the following +format: + +----------------------------------- +[mongodb://][user:pass@]host[:port][?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: mongodb + hosts: ["localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: mongodb + hosts: ["mongodb://myuser:mypass@localhost:40001", "otherhost:40001"] +---------------------------------------------------------------------- + +The username and password can be included in the URL or they can be set using +the respective configuration options. The credentials in the URL take precedence +over the username and password configuration options. + +[source,yaml] +---- +- module: mongodb + metricsets: ["status"] + hosts: ["localhost:27017"] + username: root + password: test +---- + +The default metricsets are `collstats`, `dbstats` and `status`. + +[float] +=== Compatibility + +The MongoDB metricsets were tested with MongoDB 3.4 and 3.0 and are expected to +work with all versions >= 2.8. + +[float] +=== MongoDB Privileges + +In order to use the metricsets, the MongoDB user specified in the module configuration needs to have certain https://docs.mongodb.com/manual/core/authorization/#privileges[privileges]. + +We recommend using the https://docs.mongodb.com/manual/reference/built-in-roles/#clusterMonitor[`clusterMonitor` role] to cover all the necessary privileges. + +You can use the following command in Mongo shell to create the privileged user (make sure you are using the `admin` db by using `db` command in Mongo shell). + +["source","js",subs="attributes"] +---- +db.createUser( + { + user: "beats", + pwd: "pass", + roles: ["clusterMonitor"] + } +) +---- + +You can use the following command in Mongo shell to grant the role to an existing user (make sure you are using the `admin` db by using `db` command in Mongo shell). + +["source","js",subs="attributes"] +---- +db.grantRolesToUser("user", ["clusterMonitor"]) +---- + + +[float] +=== Example configuration + +The MongoDB module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: mongodb + metricsets: ["dbstats", "status", "collstats", "metrics", "replstatus"] + period: 10s + enabled: true + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port]. + # The username and password can also be set using the respective configuration + # options. The credentials in the URL take precedence over the username and + # password configuration options. + hosts: ["localhost:27017"] + + # Optional SSL. By default is off. + #ssl.enabled: true + + # Mode of verification of server certificate ('none' or 'full') + #ssl.verification_mode: 'full' + + # List of root certificates for TLS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Username to use when connecting to MongoDB. Empty by default. + #username: user + + # Password to use when connecting to MongoDB. Empty by default. + #password: pass +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +include::mongodb/collstats.asciidoc[] + +include::mongodb/dbstats.asciidoc[] + +include::mongodb/metrics.asciidoc[] + +include::mongodb/replstatus.asciidoc[] + +include::mongodb/status.asciidoc[] + diff --git a/metricbeat/docs/modules/mongodb/collstats.asciidoc b/metricbeat/docs/modules/mongodb/collstats.asciidoc new file mode 100644 index 000000000000..e53f1ac05c2a --- /dev/null +++ b/metricbeat/docs/modules/mongodb/collstats.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mongodb-collstats]] +=== MongoDB collstats metricset + +include::../../../module/mongodb/collstats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mongodb/collstats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mongodb/dbstats.asciidoc b/metricbeat/docs/modules/mongodb/dbstats.asciidoc new file mode 100644 index 000000000000..e1a822a6a0cc --- /dev/null +++ b/metricbeat/docs/modules/mongodb/dbstats.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mongodb-dbstats]] +=== MongoDB dbstats metricset + +include::../../../module/mongodb/dbstats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mongodb/dbstats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mongodb/metrics.asciidoc b/metricbeat/docs/modules/mongodb/metrics.asciidoc new file mode 100644 index 000000000000..6c63fd964843 --- /dev/null +++ b/metricbeat/docs/modules/mongodb/metrics.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mongodb-metrics]] +=== MongoDB metrics metricset + +include::../../../module/mongodb/metrics/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mongodb/metrics/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mongodb/replstatus.asciidoc b/metricbeat/docs/modules/mongodb/replstatus.asciidoc new file mode 100644 index 000000000000..cf077c457809 --- /dev/null +++ b/metricbeat/docs/modules/mongodb/replstatus.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mongodb-replstatus]] +=== MongoDB replstatus metricset + +include::../../../module/mongodb/replstatus/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mongodb/replstatus/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mongodb/status.asciidoc b/metricbeat/docs/modules/mongodb/status.asciidoc new file mode 100644 index 000000000000..57d0c480c5f7 --- /dev/null +++ b/metricbeat/docs/modules/mongodb/status.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mongodb-status]] +=== MongoDB status metricset + +include::../../../module/mongodb/status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mongodb/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mssql.asciidoc b/metricbeat/docs/modules/mssql.asciidoc new file mode 100644 index 000000000000..4e2049cc2b89 --- /dev/null +++ b/metricbeat/docs/modules/mssql.asciidoc @@ -0,0 +1,64 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-mssql]] +[role="xpack"] +== MSSQL module + +beta[] + +This is the https://www.microsoft.com/en-us/sql-server/sql-server-2017[Microsoft SQL 2017] Metricbeat module. It is still in beta and under active development to add new Metricsets and introduce enhancements. + +[float] +== Compatibility + +The module is being tested with https://hub.docker.com/r/microsoft/mssql-server-linux/[2017 GA] version under Linux + +[float] +== Metricsets + +The following Metricsets are already included: + +[float] +=== `transaction_log` + +`transaction_log` Metricset fetches information about the operation and transaction log of each MSSQL database in the monitored instance. All data is extracted from the https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/database-related-dynamic-management-views-transact-sql?view=sql-server-2017[Database Dynamic Management Views] + +[float] +=== `performance` + +`performance` Metricset fetches information from what's commonly known as https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-2017[Performance Counters] in MSSQL. + + +[float] +=== Example configuration + +The MSSQL module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: mssql + metricsets: + - "transaction_log" + - "performance" + hosts: ["sqlserver://sa@localhost"] + period: 10s + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::mssql/performance.asciidoc[] + +include::mssql/transaction_log.asciidoc[] + diff --git a/metricbeat/docs/modules/mssql/performance.asciidoc b/metricbeat/docs/modules/mssql/performance.asciidoc new file mode 100644 index 000000000000..214ae0455fa2 --- /dev/null +++ b/metricbeat/docs/modules/mssql/performance.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mssql-performance]] +=== MSSQL performance metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/mssql/performance/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mssql/transaction_log.asciidoc b/metricbeat/docs/modules/mssql/transaction_log.asciidoc new file mode 100644 index 000000000000..cf0bb4f35ebc --- /dev/null +++ b/metricbeat/docs/modules/mssql/transaction_log.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mssql-transaction_log]] +=== MSSQL transaction_log metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/mssql/transaction_log/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/mssql/transaction_log/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/munin.asciidoc b/metricbeat/docs/modules/munin.asciidoc new file mode 100644 index 000000000000..babe65941265 --- /dev/null +++ b/metricbeat/docs/modules/munin.asciidoc @@ -0,0 +1,53 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-munin]] +== Munin module + +This is the munin module. + +The default metricset is `node`. + +[float] +=== Compatibility + +Munin module should be compatible with any implementation of the munin network +protocol (http://guide.munin-monitoring.org/en/latest/master/network-protocol.html), +it is tested with munin node 2.0. + + +[float] +=== Example configuration + +The Munin module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: munin + metricsets: ["node"] + enabled: true + period: 10s + hosts: ["localhost:4949"] + + # List of plugins to collect metrics from, by default it collects from + # all the available ones. + #munin.plugins: [] + + # If set to true, it sanitizes fields names in concordance with munin + # implementation (all characters that are not alphanumeric, or underscore + # are replaced by underscores). + #munin.sanitize: false +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::munin/node.asciidoc[] + diff --git a/metricbeat/docs/modules/munin/node.asciidoc b/metricbeat/docs/modules/munin/node.asciidoc new file mode 100644 index 000000000000..de4e7296b86a --- /dev/null +++ b/metricbeat/docs/modules/munin/node.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-munin-node]] +=== Munin node metricset + +include::../../../module/munin/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/munin/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc new file mode 100644 index 000000000000..549e8e1dafcc --- /dev/null +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -0,0 +1,94 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-mysql]] +== MySQL module + +This module periodically fetches metrics from https://www.mysql.com/[MySQL] +servers. + +The default metricset is `status`. + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use a MySQL Data Source Name (DSN) +of the following format: + +---- +[username[:password]@][protocol[(address)]]/ +---- + +You can also separately specify the username and password using the respective +configuration options. Usernames and passwords specified in the DSN take +precedence over those specified in the `username` and `password` config options. + +---- +- module: mysql + metricsets: ["status"] + hosts: ["tcp(127.0.0.1:3306)/"] + username: root + password: secret +---- + +[float] +=== Compatibility + +The mysql MetricSets were tested with MySQL and Percona 5.7 and 8.0 and are expected +to work with all versions >= 5.7.0. +It is also tested with MariaDB 10.2, 10.3 and 10.4. + +[float] +=== Dashboard + +The mysql module comes with a predefined dashboard. For example: + +image::./images/metricbeat-mysql.png[] + + +[float] +=== Example configuration + +The MySQL module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: mysql + metricsets: + - "status" + # - "galera_status" + period: 10s + + # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" + # or "unix(/var/lib/mysql/mysql.sock)/", + # or another DSN format supported by . + # The username and password can either be set in the DSN or using the username + # and password config options. Those specified in the DSN take precedence. + hosts: ["root:secret@tcp(127.0.0.1:3306)/"] + + # Username of hosts. Empty by default. + #username: root + + # Password of hosts. Empty by default. + #password: secret + + # By setting raw to true, all raw fields from the status metricset will be added to the event. + #raw: false +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::mysql/galera_status.asciidoc[] + +include::mysql/status.asciidoc[] + diff --git a/metricbeat/docs/modules/mysql/galera_status.asciidoc b/metricbeat/docs/modules/mysql/galera_status.asciidoc new file mode 100644 index 000000000000..c0d95bf5e893 --- /dev/null +++ b/metricbeat/docs/modules/mysql/galera_status.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mysql-galera_status]] +=== MySQL galera_status metricset + +beta[] + +include::../../../module/mysql/galera_status/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mysql/galera_status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mysql/status.asciidoc b/metricbeat/docs/modules/mysql/status.asciidoc new file mode 100644 index 000000000000..ff19f0d6a2d3 --- /dev/null +++ b/metricbeat/docs/modules/mysql/status.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mysql-status]] +=== MySQL status metricset + +include::../../../module/mysql/status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mysql/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/nats.asciidoc b/metricbeat/docs/modules/nats.asciidoc new file mode 100644 index 000000000000..75b3b8a33062 --- /dev/null +++ b/metricbeat/docs/modules/nats.asciidoc @@ -0,0 +1,65 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-nats]] +== Nats module + +This is the Nats module. The Nats module uses https://nats.io/documentation/managing_the_server/monitoring/[Nats monitoring server APIs] to collect metrics. + +The default metricsets are `stats`, `connections`, `routes` and `subscriptions`. + +[float] +=== Compatibility + +The Nats module is tested with Nats 1.3.0 and 2.0.4 + + +[float] +=== Dashboard + +The Nats module comes with a predefined dashboard. For example: + +image::./images/metricbeat_nats_dashboard.png[] + + +[float] +=== Example configuration + +The Nats module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: nats + metricsets: ["connections", "routes", "stats", "subscriptions"] + period: 10s + hosts: ["localhost:8222"] + #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" + #routes.metrics_path: "/routez" + #subscriptions.metrics_path: "/subsz" +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::nats/connections.asciidoc[] + +include::nats/routes.asciidoc[] + +include::nats/stats.asciidoc[] + +include::nats/subscriptions.asciidoc[] + diff --git a/metricbeat/docs/modules/nats/connections.asciidoc b/metricbeat/docs/modules/nats/connections.asciidoc new file mode 100644 index 000000000000..a64a2a46cf5d --- /dev/null +++ b/metricbeat/docs/modules/nats/connections.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nats-connections]] +=== Nats connections metricset + +include::../../../module/nats/connections/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/connections/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/nats/routes.asciidoc b/metricbeat/docs/modules/nats/routes.asciidoc new file mode 100644 index 000000000000..e85565194ff2 --- /dev/null +++ b/metricbeat/docs/modules/nats/routes.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nats-routes]] +=== Nats routes metricset + +include::../../../module/nats/routes/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/routes/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/nats/stats.asciidoc b/metricbeat/docs/modules/nats/stats.asciidoc new file mode 100644 index 000000000000..da5a1abcfb8e --- /dev/null +++ b/metricbeat/docs/modules/nats/stats.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nats-stats]] +=== Nats stats metricset + +include::../../../module/nats/stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/nats/subscriptions.asciidoc b/metricbeat/docs/modules/nats/subscriptions.asciidoc new file mode 100644 index 000000000000..53f36c4a4529 --- /dev/null +++ b/metricbeat/docs/modules/nats/subscriptions.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nats-subscriptions]] +=== Nats subscriptions metricset + +include::../../../module/nats/subscriptions/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/subscriptions/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/nginx.asciidoc b/metricbeat/docs/modules/nginx.asciidoc new file mode 100644 index 000000000000..58bc584bb849 --- /dev/null +++ b/metricbeat/docs/modules/nginx.asciidoc @@ -0,0 +1,59 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-nginx]] +== Nginx module + +This module periodically fetches metrics from https://nginx.org/[Nginx] servers. + +The default metricset is `stubstatus`. + + +[float] +=== Compatibility + +The Nginx metricsets were tested with Nginx 1.9 and are expected to work with all version +>= 1.9. + +[float] +=== Dashboard + +The nginx module comes with a predefined dashboard. For example: + +image::./images/metricbeat-nginx.png[] + + +[float] +=== Example configuration + +The Nginx module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: nginx + metricsets: ["stubstatus"] + enabled: true + period: 10s + + # Nginx hosts + hosts: ["http://127.0.0.1"] + + # Path to server status. Default nginx_status + server_status_path: "nginx_status" +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::nginx/stubstatus.asciidoc[] + diff --git a/metricbeat/docs/modules/nginx/stubstatus.asciidoc b/metricbeat/docs/modules/nginx/stubstatus.asciidoc new file mode 100644 index 000000000000..c48feb758389 --- /dev/null +++ b/metricbeat/docs/modules/nginx/stubstatus.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nginx-stubstatus]] +=== Nginx stubstatus metricset + +include::../../../module/nginx/stubstatus/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nginx/stubstatus/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/oracle.asciidoc b/metricbeat/docs/modules/oracle.asciidoc new file mode 100644 index 000000000000..9cab2b41848a --- /dev/null +++ b/metricbeat/docs/modules/oracle.asciidoc @@ -0,0 +1,81 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-oracle]] +[role="xpack"] +== Oracle module + +beta[] + +This is the https://www.oracle.com[Oracle] module for Metricbeat. It is under active development with feedback from the community. A single Metricset for Tablespace monitoring is added so the community can start gathering metrics from their nodes and contributing to the module. + +[float] +== Compatibility +Oracle Metricbeat module is being tested version 12c R2 by using the store/oracle/database-enterprise:12.2.0.1 Docker image downloaded directly from the https://hub.docker.com/_/oracle-database-enterprise-edition[Oracle Docker Hub repository] +which is based on 5.0.13-arch1-1-ARCH Arch Linux. This is important, the module has only been tested with the mentioned image in Linux environments. + +OCI Instant Client is also required and the module has been tested by using version 18.5. See below for more information. + +[float] +== Dashboard +An overview dashboard for Kibana is already included: + +image::./images/metricbeat-oracle-overview.png[] + +[float] + +== Requirements +Oracle database requires a special OCI connector to get connected to the database. You can find more information about it in the https://www.oracle.com/database/technologies/instant-client.html[official link of Oracle's webpage]. Oracle's webpage contains a comprehensive manual about installation methods and troubleshooting. A small description is included here but follow Oracle's official documentation if you find some issue. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here]. The Oracle Technology Network License Agreement with Oracle must be accepted to download the library. + +The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. The environment variable `LD_LIBRARY_PATH` is used to set the reference required by the Metricbeat library. For example, let's say you have downloaded the library into your `Downloads` folder, you can follow the steps below to install the ZIP library on `/usr/lib` (assuming you have `wget` and a zip unpacker): + + unzip $HOME/Downloads/instantclient-basic-linux.x64-18.5.0.0.0dbru.zip -d /usr/lib + export LD_LIBRARY_PATH=/usr/lib/instantclient_18_5:$LD_LIBRARY_PATH` + +Then, Metricbeat can be launched normally if the environment variable is set. + +[float] +== Metricsets + +The following Metricset is included in the module: + +[float] +=== `tablespaces` + +Includes information about data files and temp files, grouped by Tablespace with free space available, used space, status of the data files, status of the Tablespace, etc. + + +[float] +=== Example configuration + +The Oracle module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: oracle + metricsets: ["tablespace", "performance"] + enabled: true + period: 10s + hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"] + + # username: "" + # password: "" + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::oracle/performance.asciidoc[] + +include::oracle/tablespace.asciidoc[] + diff --git a/metricbeat/docs/modules/oracle/performance.asciidoc b/metricbeat/docs/modules/oracle/performance.asciidoc new file mode 100644 index 000000000000..39fe10002fcc --- /dev/null +++ b/metricbeat/docs/modules/oracle/performance.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-oracle-performance]] +=== Oracle performance metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/oracle/performance/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/oracle/performance/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/oracle/tablespace.asciidoc b/metricbeat/docs/modules/oracle/tablespace.asciidoc new file mode 100644 index 000000000000..4e498244fea7 --- /dev/null +++ b/metricbeat/docs/modules/oracle/tablespace.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-oracle-tablespace]] +=== Oracle tablespace metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/oracle/tablespace/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/oracle/tablespace/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc new file mode 100644 index 000000000000..902bc104d026 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -0,0 +1,75 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-php_fpm]] +== PHP_FPM module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +The default metricset is `pool`. + +[float] +=== Module-specific configuration notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): + +---- +nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +---- + + +[float] +=== Compatibility + +The PHP_FPM metricsets were tested with PHP 7.1.1 and are expected to +work with all versions >= 5. + + +[float] +=== Example configuration + +The PHP_FPM module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: php_fpm + metricsets: + - pool + #- process + enabled: true + period: 10s + status_path: "/status" + hosts: ["localhost:8080"] +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::php_fpm/pool.asciidoc[] + +include::php_fpm/process.asciidoc[] + diff --git a/metricbeat/docs/modules/php_fpm/pool.asciidoc b/metricbeat/docs/modules/php_fpm/pool.asciidoc new file mode 100644 index 000000000000..fec66304d12a --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/pool.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-php_fpm-pool]] +=== PHP_FPM pool metricset + +include::../../../module/php_fpm/pool/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/pool/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/php_fpm/process.asciidoc b/metricbeat/docs/modules/php_fpm/process.asciidoc new file mode 100644 index 000000000000..3fd9483af42e --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/process.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-php_fpm-process]] +=== PHP_FPM process metricset + +include::../../../module/php_fpm/process/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/process/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc new file mode 100644 index 000000000000..1958a3b314e3 --- /dev/null +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -0,0 +1,124 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-postgresql]] +== PostgreSQL module + +This module periodically fetches metrics from +https://www.postgresql.org/[PostgreSQL] servers. + +Default metricsets are `activity`, `bgwriter` and `database`. + + +[float] +=== Dashboard + +The PostgreSQL module comes with a predefined dashboard showing databse related metrics. For example: + +image::./images/metricbeat-postgresql-overview.png[] + + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use Postgres URLs of the following +format: + +----------------------------------- +[postgres://][user:pass@]host[:port][?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"] +---------------------------------------------------------------------- + +You can also separately specify the username and password using the respective +configuration options. Usernames and passwords specified in the URL take +precedence over those specified in the `username` and `password` config options. + +[source,yaml] +---- +- module: postgresql + metricsets: ["status"] + hosts: ["postgres://localhost:5432"] + username: root + password: test +---- + +[float] +=== Compatibility + +This module was tested with PostgreSQL 9.5.3 and is expected to work with all +versions >= 9. + + +[float] +=== Example configuration + +The PostgreSQL module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: postgresql + enabled: true + metricsets: + # Stats about every PostgreSQL database + - database + + # Stats about the background writer process's activity + - bgwriter + + # Stats about every PostgreSQL process + - activity + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost:5432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::postgresql/activity.asciidoc[] + +include::postgresql/bgwriter.asciidoc[] + +include::postgresql/database.asciidoc[] + +include::postgresql/statement.asciidoc[] + diff --git a/metricbeat/docs/modules/postgresql/activity.asciidoc b/metricbeat/docs/modules/postgresql/activity.asciidoc new file mode 100644 index 000000000000..92fc5ae3d10a --- /dev/null +++ b/metricbeat/docs/modules/postgresql/activity.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-postgresql-activity]] +=== PostgreSQL activity metricset + +include::../../../module/postgresql/activity/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/postgresql/activity/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/postgresql/bgwriter.asciidoc b/metricbeat/docs/modules/postgresql/bgwriter.asciidoc new file mode 100644 index 000000000000..94a55fb3afcb --- /dev/null +++ b/metricbeat/docs/modules/postgresql/bgwriter.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-postgresql-bgwriter]] +=== PostgreSQL bgwriter metricset + +include::../../../module/postgresql/bgwriter/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/postgresql/bgwriter/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/postgresql/database.asciidoc b/metricbeat/docs/modules/postgresql/database.asciidoc new file mode 100644 index 000000000000..b1598a74ee00 --- /dev/null +++ b/metricbeat/docs/modules/postgresql/database.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-postgresql-database]] +=== PostgreSQL database metricset + +include::../../../module/postgresql/database/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/postgresql/database/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/postgresql/statement.asciidoc b/metricbeat/docs/modules/postgresql/statement.asciidoc new file mode 100644 index 000000000000..dd87c900be3a --- /dev/null +++ b/metricbeat/docs/modules/postgresql/statement.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-postgresql-statement]] +=== PostgreSQL statement metricset + +include::../../../module/postgresql/statement/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/postgresql/statement/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc new file mode 100644 index 000000000000..46a5f672939c --- /dev/null +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -0,0 +1,55 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-prometheus]] +== Prometheus module + +[[prometheus-module]] +This module periodically scrapes metrics from +https://prometheus.io/docs/instrumenting/exporters/[Prometheus exporters]. + +[float] +=== Dashboard + +The Prometheus module comes with a predefined dashboard for Prometheus specific stats. For example: + +image::./images/metricbeat-prometheus-overview.png[] + + + + +[float] +=== Example configuration + +The Prometheus module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::prometheus/collector.asciidoc[] + diff --git a/metricbeat/docs/modules/prometheus/collector.asciidoc b/metricbeat/docs/modules/prometheus/collector.asciidoc new file mode 100644 index 000000000000..5bde631b91ba --- /dev/null +++ b/metricbeat/docs/modules/prometheus/collector.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-prometheus-collector]] +=== Prometheus collector metricset + +include::../../../module/prometheus/collector/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/prometheus/collector/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/rabbitmq.asciidoc b/metricbeat/docs/modules/rabbitmq.asciidoc new file mode 100644 index 000000000000..fdb2eae18d0a --- /dev/null +++ b/metricbeat/docs/modules/rabbitmq.asciidoc @@ -0,0 +1,66 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-rabbitmq]] +== RabbitMQ module + +The RabbitMQ module uses http://www.rabbitmq.com/management.html[HTTP API] created by the management plugin to collect metrics. + +The default metricsets are `connection`, `node`, `queue` and `exchange`. + +If `management.path_prefix` is set in RabbitMQ configuration, `management_path_prefix` has to be set to the same value in this module configuration. + +[float] +=== Compatibility + +The rabbitmq module is fully tested with RabbitMQ 3.7.4 and it should be compatible with any version supporting the management plugin (which needs to be installed and enabled). Exchange metricset is also tested with 3.6.0, 3.6.5 and 3.7.14 + + +[float] +=== Example configuration + +The RabbitMQ module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: rabbitmq + metricsets: ["node", "queue", "connection"] + enabled: true + period: 10s + hosts: ["localhost:15672"] + + # Management path prefix, if `management.path_prefix` is set in RabbitMQ + # configuration, it has to be set to the same value. + #management_path_prefix: "" + + #username: guest + #password: guest +---- + +This module supports TLS connections when using `ssl` config field, as described in <>. +It also supports the options described in <>. + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::rabbitmq/connection.asciidoc[] + +include::rabbitmq/exchange.asciidoc[] + +include::rabbitmq/node.asciidoc[] + +include::rabbitmq/queue.asciidoc[] + diff --git a/metricbeat/docs/modules/rabbitmq/connection.asciidoc b/metricbeat/docs/modules/rabbitmq/connection.asciidoc new file mode 100644 index 000000000000..e90f72cdec0b --- /dev/null +++ b/metricbeat/docs/modules/rabbitmq/connection.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-rabbitmq-connection]] +=== RabbitMQ connection metricset + +include::../../../module/rabbitmq/connection/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/rabbitmq/connection/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/rabbitmq/exchange.asciidoc b/metricbeat/docs/modules/rabbitmq/exchange.asciidoc new file mode 100644 index 000000000000..ba0b3c4c98a0 --- /dev/null +++ b/metricbeat/docs/modules/rabbitmq/exchange.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-rabbitmq-exchange]] +=== RabbitMQ exchange metricset + +include::../../../module/rabbitmq/exchange/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/rabbitmq/exchange/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/rabbitmq/node.asciidoc b/metricbeat/docs/modules/rabbitmq/node.asciidoc new file mode 100644 index 000000000000..56c5bcf10c52 --- /dev/null +++ b/metricbeat/docs/modules/rabbitmq/node.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-rabbitmq-node]] +=== RabbitMQ node metricset + +include::../../../module/rabbitmq/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/rabbitmq/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/rabbitmq/queue.asciidoc b/metricbeat/docs/modules/rabbitmq/queue.asciidoc new file mode 100644 index 000000000000..83fe6c38ce13 --- /dev/null +++ b/metricbeat/docs/modules/rabbitmq/queue.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-rabbitmq-queue]] +=== RabbitMQ queue metricset + +include::../../../module/rabbitmq/queue/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/rabbitmq/queue/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/redis.asciidoc b/metricbeat/docs/modules/redis.asciidoc new file mode 100644 index 000000000000..b5c7f20364fa --- /dev/null +++ b/metricbeat/docs/modules/redis.asciidoc @@ -0,0 +1,97 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-redis]] +== Redis module + +This module periodically fetches metrics from http://redis.io/[Redis] servers. + +The defaut metricsets are `info` and `keyspace`. + +[float] +=== Module-specific configuration notes + +The Redis module has these additional config options: + +*`hosts`*:: URLs that are used to connect to Redis. +URL format: +redis://[:password@]host[:port][/db-number][?option=value] +redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]] +*`password`*:: The password to authenticate, by default it's empty. +*`idle_timeout`*:: The duration to remain idle before closing connections. If + the value is zero, then idle connections are not closed. The default value + is 2 times the module period to allow a connection to be reused across + fetches. The `idle_timeout` should be set to less than the server's connection + timeout. +*`network`*:: The network type to be used for the Redis connection. The default value is + `tcp`. +*`maxconn`*:: The maximum number of concurrent connections to Redis. The default value + is 10. + + +[float] +=== Compatibility + +The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected +to work with all versions >= 3.0. + + +[float] +=== Example configuration + +The Redis module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: redis + metricsets: ["info", "keyspace"] + enabled: true + period: 10s + + # Redis hosts + hosts: ["127.0.0.1:6379"] + + # Timeout after which time a metricset should return an error + # Timeout is by default defined as period, as a fetch of a metricset + # should never take longer then period, as otherwise calls can pile up. + #timeout: 1s + + # Optional fields to be added to each event + #fields: + # datacenter: west + + # Network type to be used for redis connection. Default: tcp + #network: tcp + + # Max number of concurrent connections. Default: 10 + #maxconn: 10 + + # Filters can be used to reduce the number of fields sent. + #processors: + # - include_fields: + # fields: ["beat", "metricset", "redis.info.stats"] + + # Redis AUTH password. Empty by default. + #password: foobared +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::redis/info.asciidoc[] + +include::redis/key.asciidoc[] + +include::redis/keyspace.asciidoc[] + diff --git a/metricbeat/docs/modules/redis/info.asciidoc b/metricbeat/docs/modules/redis/info.asciidoc new file mode 100644 index 000000000000..cf13ce7f6f57 --- /dev/null +++ b/metricbeat/docs/modules/redis/info.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-redis-info]] +=== Redis info metricset + +include::../../../module/redis/info/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/redis/info/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/redis/key.asciidoc b/metricbeat/docs/modules/redis/key.asciidoc new file mode 100644 index 000000000000..b5e569957c59 --- /dev/null +++ b/metricbeat/docs/modules/redis/key.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-redis-key]] +=== Redis key metricset + +include::../../../module/redis/key/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/redis/key/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/redis/keyspace.asciidoc b/metricbeat/docs/modules/redis/keyspace.asciidoc new file mode 100644 index 000000000000..fe8420620548 --- /dev/null +++ b/metricbeat/docs/modules/redis/keyspace.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-redis-keyspace]] +=== Redis keyspace metricset + +include::../../../module/redis/keyspace/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/redis/keyspace/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/statsd.asciidoc b/metricbeat/docs/modules/statsd.asciidoc new file mode 100644 index 000000000000..4eae7c058d82 --- /dev/null +++ b/metricbeat/docs/modules/statsd.asciidoc @@ -0,0 +1,39 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-statsd]] +[role="xpack"] +== Statsd module + +beta[] + +The Statsd module is a Metricbeat module which opens a UDP port and listens for statsd metrics. + + + +[float] +=== Example configuration + +The Statsd module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: statsd + host: "localhost" + port: "8125" + enabled: false + #ttl: "30s" +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::statsd/server.asciidoc[] + diff --git a/metricbeat/docs/modules/statsd/server.asciidoc b/metricbeat/docs/modules/statsd/server.asciidoc new file mode 100644 index 000000000000..a44d850cdff2 --- /dev/null +++ b/metricbeat/docs/modules/statsd/server.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-statsd-server]] +=== Statsd server metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/statsd/server/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/statsd/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc new file mode 100644 index 000000000000..d6e952dc7883 --- /dev/null +++ b/metricbeat/docs/modules/system.asciidoc @@ -0,0 +1,183 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-system]] +== System module + +The System module allows you to monitor your servers. Because the System module +always applies to the local server, the `hosts` config option is not needed. + +The default metricsets are `cpu`, `load`, `memory`, `network`, `process`, and +`process_summary`. To disable a default metricset, comment it out in the +`modules.d/system.yml` configuration file. If _all_ metricsets are commented out +and the System module is enabled, {beatname_uc} uses the default metricsets. + +Note that certain metricsets may access `/proc` to gather process information, +and the resulting `ptrace_may_access()` call by the kernel to check for +permissions can be blocked by +https://gitlab.com/apparmor/apparmor/wikis/TechnicalDoc_Proc_and_ptrace[AppArmor +and other LSM software], even though the System module doesn't use `ptrace` +directly. + +[float] +=== Dashboard + +The System module comes with a predefined dashboard. For example: + +image::./images/metricbeat_system_dashboard.png[] + + +[float] +=== Example configuration + +The System module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: system + metricsets: + - cpu # CPU usage + #- load # CPU load averages + - memory # Memory usage + - network # Network IO + - process # Per process metrics + - process_summary # Process summary + - uptime # System Uptime + - socket_summary # Socket summary + #- core # Per CPU core usage + #- diskio # Disk IO + #- filesystem # File system usage for each mountpoint + #- fsstat # File system summary metrics + #- raid # Raid + #- socket # Sockets and connection info (linux only) + enabled: true + period: 10s + processes: ['.*'] + + # Configure the metric types that are included by these metricsets. + cpu.metrics: ["percentages"] # The other available options are normalized_percentages and ticks. + core.metrics: ["percentages"] # The other available option is ticks. + + # A list of filesystem types to ignore. The filesystem metricset will not + # collect data from filesystems matching any of the specified types, and + # fsstats will not include data from these filesystems in its summary stats. + # If not set, types associated to virtual filesystems are automatically + # added when this information is available in the system (e.g. the list of + # `nodev` types in `/proc/filesystem`). + #filesystem.ignore_types: [] + + # These options allow you to filter out all processes that are not + # in the top N by CPU or memory, in order to reduce the number of documents created. + # If both the `by_cpu` and `by_memory` options are used, the union of the two sets + # is included. + #process.include_top_n: + + # Set to false to disable this feature and include all processes + #enabled: true + + # How many processes to include from the top by CPU. The processes are sorted + # by the `system.process.cpu.total.pct` field. + #by_cpu: 0 + + # How many processes to include from the top by memory. The processes are sorted + # by the `system.process.memory.rss.bytes` field. + #by_memory: 0 + + # If false, cmdline of a process is not cached. + #process.cmdline.cache.enabled: true + + # Enable collection of cgroup metrics from processes on Linux. + #process.cgroups.enabled: true + + # A list of regular expressions used to whitelist environment variables + # reported with the process metricset's events. Defaults to empty. + #process.env.whitelist: [] + + # Include the cumulative CPU tick values with the process metrics. Defaults + # to false. + #process.include_cpu_ticks: false + + # Raid mount point to monitor + #raid.mount_point: '/' + + # Configure reverse DNS lookup on remote IP addresses in the socket metricset. + #socket.reverse_lookup.enabled: false + #socket.reverse_lookup.success_ttl: 60s + #socket.reverse_lookup.failure_ttl: 60s + + # Diskio configurations + #diskio.include_devices: [] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + +include::system/core.asciidoc[] + +include::system/cpu.asciidoc[] + +include::system/diskio.asciidoc[] + +include::system/entropy.asciidoc[] + +include::system/filesystem.asciidoc[] + +include::system/fsstat.asciidoc[] + +include::system/load.asciidoc[] + +include::system/memory.asciidoc[] + +include::system/network.asciidoc[] + +include::system/process.asciidoc[] + +include::system/process_summary.asciidoc[] + +include::system/raid.asciidoc[] + +include::system/service.asciidoc[] + +include::system/socket.asciidoc[] + +include::system/socket_summary.asciidoc[] + +include::system/uptime.asciidoc[] + diff --git a/metricbeat/docs/modules/system/core.asciidoc b/metricbeat/docs/modules/system/core.asciidoc new file mode 100644 index 000000000000..5dad70a74251 --- /dev/null +++ b/metricbeat/docs/modules/system/core.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-core]] +=== System core metricset + +include::../../../module/system/core/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/core/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/cpu.asciidoc b/metricbeat/docs/modules/system/cpu.asciidoc new file mode 100644 index 000000000000..7202e10bdab0 --- /dev/null +++ b/metricbeat/docs/modules/system/cpu.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-cpu]] +=== System cpu metricset + +include::../../../module/system/cpu/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/cpu/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/diskio.asciidoc b/metricbeat/docs/modules/system/diskio.asciidoc new file mode 100644 index 000000000000..192c0320b437 --- /dev/null +++ b/metricbeat/docs/modules/system/diskio.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-diskio]] +=== System diskio metricset + +include::../../../module/system/diskio/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/diskio/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/entropy.asciidoc b/metricbeat/docs/modules/system/entropy.asciidoc new file mode 100644 index 000000000000..2d4abcb86d1f --- /dev/null +++ b/metricbeat/docs/modules/system/entropy.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-entropy]] +=== System entropy metricset + +include::../../../module/system/entropy/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/entropy/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/filesystem.asciidoc b/metricbeat/docs/modules/system/filesystem.asciidoc new file mode 100644 index 000000000000..b508394f5aba --- /dev/null +++ b/metricbeat/docs/modules/system/filesystem.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-filesystem]] +=== System filesystem metricset + +include::../../../module/system/filesystem/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/filesystem/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/fsstat.asciidoc b/metricbeat/docs/modules/system/fsstat.asciidoc new file mode 100644 index 000000000000..3abfce2b88c7 --- /dev/null +++ b/metricbeat/docs/modules/system/fsstat.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-fsstat]] +=== System fsstat metricset + +include::../../../module/system/fsstat/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/fsstat/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/load.asciidoc b/metricbeat/docs/modules/system/load.asciidoc new file mode 100644 index 000000000000..d9e7b67ffad3 --- /dev/null +++ b/metricbeat/docs/modules/system/load.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-load]] +=== System load metricset + +include::../../../module/system/load/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/load/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/memory.asciidoc b/metricbeat/docs/modules/system/memory.asciidoc new file mode 100644 index 000000000000..2210a930ab6d --- /dev/null +++ b/metricbeat/docs/modules/system/memory.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-memory]] +=== System memory metricset + +include::../../../module/system/memory/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/network.asciidoc b/metricbeat/docs/modules/system/network.asciidoc new file mode 100644 index 000000000000..3b3f51d6ed31 --- /dev/null +++ b/metricbeat/docs/modules/system/network.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-network]] +=== System network metricset + +include::../../../module/system/network/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/network/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/process.asciidoc b/metricbeat/docs/modules/system/process.asciidoc new file mode 100644 index 000000000000..13b01ed90b51 --- /dev/null +++ b/metricbeat/docs/modules/system/process.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-process]] +=== System process metricset + +include::../../../module/system/process/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/process/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/process_summary.asciidoc b/metricbeat/docs/modules/system/process_summary.asciidoc new file mode 100644 index 000000000000..cf2145457a2f --- /dev/null +++ b/metricbeat/docs/modules/system/process_summary.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-process_summary]] +=== System process_summary metricset + +include::../../../module/system/process_summary/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/process_summary/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/raid.asciidoc b/metricbeat/docs/modules/system/raid.asciidoc new file mode 100644 index 000000000000..cf8092e3954e --- /dev/null +++ b/metricbeat/docs/modules/system/raid.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-raid]] +=== System raid metricset + +include::../../../module/system/raid/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/raid/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/service.asciidoc b/metricbeat/docs/modules/system/service.asciidoc new file mode 100644 index 000000000000..0f7fc1899808 --- /dev/null +++ b/metricbeat/docs/modules/system/service.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-service]] +=== System service metricset + +beta[] + +include::../../../module/system/service/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/service/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/socket.asciidoc b/metricbeat/docs/modules/system/socket.asciidoc new file mode 100644 index 000000000000..b8d8f01327d3 --- /dev/null +++ b/metricbeat/docs/modules/system/socket.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-socket]] +=== System socket metricset + +include::../../../module/system/socket/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/socket/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/system/socket_summary.asciidoc b/metricbeat/docs/modules/system/socket_summary.asciidoc new file mode 100644 index 000000000000..0b3f4740a2d5 --- /dev/null +++ b/metricbeat/docs/modules/system/socket_summary.asciidoc @@ -0,0 +1,16 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-socket_summary]] +=== System socket_summary metricset + +include::../../../module/system/socket_summary/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/system/uptime.asciidoc b/metricbeat/docs/modules/system/uptime.asciidoc new file mode 100644 index 000000000000..97d5bf5947a2 --- /dev/null +++ b/metricbeat/docs/modules/system/uptime.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-system-uptime]] +=== System uptime metricset + +include::../../../module/system/uptime/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/uptime/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/tomcat.asciidoc b/metricbeat/docs/modules/tomcat.asciidoc new file mode 100644 index 000000000000..37910f2b8d47 --- /dev/null +++ b/metricbeat/docs/modules/tomcat.asciidoc @@ -0,0 +1,58 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-tomcat]] +[role="xpack"] +== Tomcat module + +beta[] + +This module periodically fetches JMX metrics from Apache Tomcat. + +[float] +=== Compatibility +The module has been tested with Tomcat 7.0.24 and 9.0.24. Other versions are expected to work. + +[float] +=== Usage +The Tomcat module requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. + + +[float] +=== Example configuration + +The Tomcat module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: tomcat + metricsets: ['threading', 'cache', 'memory', 'requests'] + period: 10s + hosts: ['localhost:8080'] + path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::tomcat/cache.asciidoc[] + +include::tomcat/memory.asciidoc[] + +include::tomcat/requests.asciidoc[] + +include::tomcat/threading.asciidoc[] + diff --git a/metricbeat/docs/modules/tomcat/cache.asciidoc b/metricbeat/docs/modules/tomcat/cache.asciidoc new file mode 100644 index 000000000000..ddddaae045e8 --- /dev/null +++ b/metricbeat/docs/modules/tomcat/cache.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-tomcat-cache]] +=== Tomcat cache metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/tomcat/cache/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/tomcat/cache/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/tomcat/memory.asciidoc b/metricbeat/docs/modules/tomcat/memory.asciidoc new file mode 100644 index 000000000000..1d850e840cd2 --- /dev/null +++ b/metricbeat/docs/modules/tomcat/memory.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-tomcat-memory]] +=== Tomcat memory metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/tomcat/memory/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/tomcat/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/tomcat/requests.asciidoc b/metricbeat/docs/modules/tomcat/requests.asciidoc new file mode 100644 index 000000000000..8f2db18d4d43 --- /dev/null +++ b/metricbeat/docs/modules/tomcat/requests.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-tomcat-requests]] +=== Tomcat requests metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/tomcat/requests/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/tomcat/requests/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/tomcat/threading.asciidoc b/metricbeat/docs/modules/tomcat/threading.asciidoc new file mode 100644 index 000000000000..4d356cb2a9a1 --- /dev/null +++ b/metricbeat/docs/modules/tomcat/threading.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-tomcat-threading]] +=== Tomcat threading metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/tomcat/threading/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/tomcat/threading/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/traefik.asciidoc b/metricbeat/docs/modules/traefik.asciidoc new file mode 100644 index 000000000000..a9188591168b --- /dev/null +++ b/metricbeat/docs/modules/traefik.asciidoc @@ -0,0 +1,39 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-traefik]] +== traefik module + +This module periodically fetches metrics from a https://traefik.io/[Traefik] +instance. The Traefik instance must be configured to expose it's HTTP API. + +=== Compatibility + +The Traefik metricsets were tested with Traefik 1.6. + + +[float] +=== Example configuration + +The traefik module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: traefik + metricsets: ["health"] + period: 10s + hosts: ["localhost:8080"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::traefik/health.asciidoc[] + diff --git a/metricbeat/docs/modules/traefik/health.asciidoc b/metricbeat/docs/modules/traefik/health.asciidoc new file mode 100644 index 000000000000..1afcf32d6439 --- /dev/null +++ b/metricbeat/docs/modules/traefik/health.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-traefik-health]] +=== traefik health metricset + +include::../../../module/traefik/health/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/traefik/health/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/uwsgi.asciidoc b/metricbeat/docs/modules/uwsgi.asciidoc new file mode 100644 index 000000000000..bb2234abe09c --- /dev/null +++ b/metricbeat/docs/modules/uwsgi.asciidoc @@ -0,0 +1,52 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-uwsgi]] +== uwsgi module + +This is the uwsgi module. By default collects the `stats` metricset, using +http://uwsgi-docs.readthedocs.io/en/latest/StatsServer.html[StatsServer]. + +[float] +=== Module-specific configuration notes + +The uWSGI module has these additional config options: + +*`hosts`*:: host URLs to get data from (e.g: `tcp://127.0.0.1:9191`). + Can obtain data from 3 types of schemes: tcp (tcp://ip:port), unix socket (unix:///tmp/uwsgi.sock) + and http/https server (http://ip:port) + +[float] +=== Dashboard + +The uwsgi module comes with a predefined dashboard. For example: + +image::./images/uwsgi_dashboard.png[] + + +[float] +=== Example configuration + +The uwsgi module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: uwsgi + metricsets: ["status"] + enable: true + period: 10s + hosts: ["tcp://127.0.0.1:9191"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::uwsgi/status.asciidoc[] + diff --git a/metricbeat/docs/modules/uwsgi/status.asciidoc b/metricbeat/docs/modules/uwsgi/status.asciidoc new file mode 100644 index 000000000000..d7497a9c2e90 --- /dev/null +++ b/metricbeat/docs/modules/uwsgi/status.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-uwsgi-status]] +=== uwsgi status metricset + +include::../../../module/uwsgi/status/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/uwsgi/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc new file mode 100644 index 000000000000..b426ea76ba9d --- /dev/null +++ b/metricbeat/docs/modules/vsphere.asciidoc @@ -0,0 +1,62 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-vsphere]] +== vSphere module + +beta[] + +The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5. + +By default it enables the metricsets `datastore`, `host` and `virtualmachine`. + +[float] +=== Dashboard + +The vsphere module comes with a predefined dashboard. For example: + +image::./images/metricbeat_vsphere_dashboard.png[] +image::./images/metricbeat_vsphere_vm_dashboard.png[] + + +[float] +=== Example configuration + +The vSphere module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: vsphere + enabled: true + metricsets: ["datastore", "host", "virtualmachine"] + period: 10s + hosts: ["https://localhost/sdk"] + + username: "user" + password: "password" + # If insecure is true, don't verify the server's certificate chain + insecure: false + # Get custom fields when using virtualmachine metric set. Default false. + # get_custom_fields: false +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::vsphere/datastore.asciidoc[] + +include::vsphere/host.asciidoc[] + +include::vsphere/virtualmachine.asciidoc[] + diff --git a/metricbeat/docs/modules/vsphere/datastore.asciidoc b/metricbeat/docs/modules/vsphere/datastore.asciidoc new file mode 100644 index 000000000000..7b68437b5ce1 --- /dev/null +++ b/metricbeat/docs/modules/vsphere/datastore.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-vsphere-datastore]] +=== vSphere datastore metricset + +beta[] + +include::../../../module/vsphere/datastore/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/vsphere/datastore/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/vsphere/host.asciidoc b/metricbeat/docs/modules/vsphere/host.asciidoc new file mode 100644 index 000000000000..583bdb2e4e6f --- /dev/null +++ b/metricbeat/docs/modules/vsphere/host.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-vsphere-host]] +=== vSphere host metricset + +beta[] + +include::../../../module/vsphere/host/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/vsphere/host/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/vsphere/virtualmachine.asciidoc b/metricbeat/docs/modules/vsphere/virtualmachine.asciidoc new file mode 100644 index 000000000000..59ac57af1f63 --- /dev/null +++ b/metricbeat/docs/modules/vsphere/virtualmachine.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-vsphere-virtualmachine]] +=== vSphere virtualmachine metricset + +beta[] + +include::../../../module/vsphere/virtualmachine/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/vsphere/virtualmachine/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc new file mode 100644 index 000000000000..e343a359dccd --- /dev/null +++ b/metricbeat/docs/modules/windows.asciidoc @@ -0,0 +1,51 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-windows]] +== Windows module + +This is the Windows module. It collects metrics from Windows systems, +by default metricset `service` is enabled. + + +[float] +=== Example configuration + +The Windows module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: windows + metricsets: ["perfmon"] + enabled: true + period: 10s + perfmon.ignore_non_existent_counters: false + perfmon.group_measurements_by_instance: false + perfmon.counters: + # - instance_label: processor.name + # instance_name: total + # measurement_label: processor.time.total.pct + # query: '\Processor Information(_Total)\% Processor Time' + +- module: windows + metricsets: ["service"] + enabled: true + period: 60s +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::windows/perfmon.asciidoc[] + +include::windows/service.asciidoc[] + diff --git a/metricbeat/docs/modules/windows/perfmon.asciidoc b/metricbeat/docs/modules/windows/perfmon.asciidoc new file mode 100644 index 000000000000..f3f53901205a --- /dev/null +++ b/metricbeat/docs/modules/windows/perfmon.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-windows-perfmon]] +=== Windows perfmon metricset + +beta[] + +include::../../../module/windows/perfmon/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/windows/perfmon/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/windows/service.asciidoc b/metricbeat/docs/modules/windows/service.asciidoc new file mode 100644 index 000000000000..d97cd83a24de --- /dev/null +++ b/metricbeat/docs/modules/windows/service.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-windows-service]] +=== Windows service metricset + +include::../../../module/windows/service/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/windows/service/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc new file mode 100644 index 000000000000..ceb2dc618423 --- /dev/null +++ b/metricbeat/docs/modules/zookeeper.asciidoc @@ -0,0 +1,57 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-zookeeper]] +== ZooKeeper module + +The ZooKeeper module fetches statistics from the ZooKeeper service. The default +metricset is `mntr` and `server`. + +[float] +=== Compatibility + +The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version +>= 3.4.0. Versions prior to 3.4 do not support the `mntr` command. + +[float] +=== Dashboard + +The Zookeeper module comes with a predefined dashboard: + +image::./images/metricbeat-zookeeper.png[] + + +[float] +=== Example configuration + +The ZooKeeper module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: zookeeper + enabled: true + metricsets: ["mntr", "server"] + period: 10s + hosts: ["localhost:2181"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::zookeeper/connection.asciidoc[] + +include::zookeeper/mntr.asciidoc[] + +include::zookeeper/server.asciidoc[] + diff --git a/metricbeat/docs/modules/zookeeper/connection.asciidoc b/metricbeat/docs/modules/zookeeper/connection.asciidoc new file mode 100644 index 000000000000..2bfcf081a5c6 --- /dev/null +++ b/metricbeat/docs/modules/zookeeper/connection.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-zookeeper-connection]] +=== ZooKeeper connection metricset + +beta[] + +include::../../../module/zookeeper/connection/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/zookeeper/connection/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/zookeeper/mntr.asciidoc b/metricbeat/docs/modules/zookeeper/mntr.asciidoc new file mode 100644 index 000000000000..7f932dc27847 --- /dev/null +++ b/metricbeat/docs/modules/zookeeper/mntr.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-zookeeper-mntr]] +=== ZooKeeper mntr metricset + +include::../../../module/zookeeper/mntr/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/zookeeper/mntr/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/zookeeper/server.asciidoc b/metricbeat/docs/modules/zookeeper/server.asciidoc new file mode 100644 index 000000000000..6b6157092ca8 --- /dev/null +++ b/metricbeat/docs/modules/zookeeper/server.asciidoc @@ -0,0 +1,22 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-zookeeper-server]] +=== ZooKeeper server metricset + +include::../../../module/zookeeper/server/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/zookeeper/server/_meta/data.json[] +---- diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 0ffeff23e859..c2246b788a02 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -77,8 +77,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" _ "github.com/elastic/beats/metricbeat/module/iis" - _ "github.com/elastic/beats/metricbeat/module/iis/webserver" - _ "github.com/elastic/beats/metricbeat/module/iis/website" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index c8f76abe443d..be927fff1fab 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -346,7 +346,6 @@ metricbeat.modules: - module: iis metricsets: - webserver - - website enabled: true period: 10s diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml index 995096a926e8..43931eccf73d 100644 --- a/metricbeat/module/iis/_meta/config.yml +++ b/metricbeat/module/iis/_meta/config.yml @@ -1,7 +1,6 @@ - module: iis metricsets: - webserver - - website enabled: true period: 10s diff --git a/metricbeat/module/iis/doc.go b/metricbeat/module/iis/doc.go index 889f4527de8d..3b68541b51c1 100644 --- a/metricbeat/module/iis/doc.go +++ b/metricbeat/module/iis/doc.go @@ -1,2 +1,19 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // Package iis is a Metricbeat module that contains MetricSets. package iis diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index 7fd087573eb6..0f5cc7f6221e 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -32,5 +32,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/iis. func AssetIis() string { - return "eJzEkDFuhTAQRHufYkTPBVyky0FMPIlWGGzZJoTbR+AQGeSk+cWfcnf0dnZ6jNw0RJICsmRHjU4kdQqwTG9RQhY/a7woALsPk7eLowIiHU2ixsBslALehc4mfTh7zGbiSd6Vt0CNj+iX8DNpHLhCatDKITF+Mv5uWsg/sUUtyPWNc1oZ7onqVPwyUzjaqFVyjdxWH+1t90+6Xa8FWI42S5DMRyu4ItoFPPXv7wAAAP//UtuoaQ==" + return "eJykz0kOwjAMheG9T/HUfS/gBXdp6QNZpINiF9TbozKJSGGFl4n1/XKLCzeFmQsQFomKxswbAQb6MdsSNk+KgwDY9zDOw5ooQGZi51T0jE4EOBnT4PrYbDF1I9/yPrEtVJzzvC6vl0qgRL6hG3tnvjJ/fmrkT/Y5NaQ8o5a14L/Rkqgn5R4AAP//OjVpIQ==" } diff --git a/metricbeat/module/iis/module.yml b/metricbeat/module/iis/module.yml new file mode 100644 index 000000000000..48b01480e3dd --- /dev/null +++ b/metricbeat/module/iis/module.yml @@ -0,0 +1,4 @@ +name: iis +metricsets: +- webserver +- website diff --git a/metricbeat/module/iis/performanceCounters.go b/metricbeat/module/iis/performanceCounters.go index fe71dcf56801..b0095d11037b 100644 --- a/metricbeat/module/iis/performanceCounters.go +++ b/metricbeat/module/iis/performanceCounters.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package iis var ( diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/reader.go index 0e597c313f02..3f74347818d9 100644 --- a/metricbeat/module/iis/reader.go +++ b/metricbeat/module/iis/reader.go @@ -20,10 +20,11 @@ package iis import ( - "github.com/elastic/beats/metricbeat/module/windows/perfmon" "regexp" "strings" + "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" diff --git a/metricbeat/module/iis/webserver/_meta/fields.yml b/metricbeat/module/iis/webserver/_meta/fields.yml index 92d0501fb34d..89041c173a82 100644 --- a/metricbeat/module/iis/webserver/_meta/fields.yml +++ b/metricbeat/module/iis/webserver/_meta/fields.yml @@ -3,9 +3,3 @@ description: > webserver release: beta - - fields: - - name: example - type: keyword - description: > - Example field diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml new file mode 100644 index 000000000000..954ce281568c --- /dev/null +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -0,0 +1,12 @@ +default: true +input: + module: windows + metricset: perfmon + defaults: + perfmon.ignore_non_existent_counters: true + perfmon.counters: + - instance_label: '' + measurement_label: network.bytes_recv_per_sec + query: '\Web Service(_Total)\Total Bytes Received' + + diff --git a/metricbeat/module/iis/webserver/webserver.go b/metricbeat/module/iis/webserver/webserver.go deleted file mode 100644 index 27a823b87399..000000000000 --- a/metricbeat/module/iis/webserver/webserver.go +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build windows - -package webserver - -import ( - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/iis" - "github.com/pkg/errors" -) - -const metricsetName = "webserver" - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("iis", metricsetName, New, mb.DefaultMetricSet()) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - mb.BaseMetricSet - reader *iis.Reader -} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Experimental("The iis webserver metricset is experimental.") - counters := iis.GetPerfCounters(metricsetName) - - reader, err := iis.NewReader(counters) - if err != nil { - return nil, errors.Wrap(err, "initialization of reader failed") - } - - return &MetricSet{ - BaseMetricSet: base, - reader: reader, - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right -// format. It publishes the event which is then forwarded to the output. In case -// of an error set the Error field of mb.Event or simply call report.Error(). -// Fetch fetches events and reports them upstream -func (m *MetricSet) Fetch(report mb.ReporterV2) error { - - // refresh performance counter list - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - // A flag is set if the second call has been executed else refresh will fail (reader.executed) - if m.reader.Executed { - err := m.reader.RefreshCounterPaths() - if err != nil { - return errors.Wrap(err, "failed retrieving counters") - } - } - - events, err := m.reader.Read(metricsetName) - if err != nil { - return errors.Wrap(err, "failed reading counters") - } - - for _, event := range events { - isOpen := report.Event(event) - if !isOpen { - break - } - } - return nil -} - -// Close will be called when metricbeat is stopped, should close the query. -func (m *MetricSet) Close() error { - err := m.reader.Close() - if err != nil { - return errors.Wrap(err, "failed to close pdh query") - } - return nil -} diff --git a/metricbeat/module/iis/webserver/webserver_test.go b/metricbeat/module/iis/webserver/webserver_test.go new file mode 100644 index 000000000000..b772274c9236 --- /dev/null +++ b/metricbeat/module/iis/webserver/webserver_test.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package webserver diff --git a/metricbeat/module/iis/website/_meta/fields.yml b/metricbeat/module/iis/website/_meta/fields.yml index c88aa44c536a..726d14ae6382 100644 --- a/metricbeat/module/iis/website/_meta/fields.yml +++ b/metricbeat/module/iis/website/_meta/fields.yml @@ -3,8 +3,4 @@ description: > website release: beta - fields: - - name: example - type: keyword - description: > - Example field + diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml new file mode 100644 index 000000000000..6c7a5249df1b --- /dev/null +++ b/metricbeat/module/iis/website/manifest.yml @@ -0,0 +1,11 @@ +default: true +input: + module: windows + metricset: perfmon + defaults: + perfmon.counters: + - instance_label: physical_disk.name + measurement_label: total_bytes_recv_per_sec + query: '\Web Service(_Total)\Total Bytes Received' + + diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go deleted file mode 100644 index f13ae1077ce8..000000000000 --- a/metricbeat/module/iis/website/website.go +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build windows - -package website - -import ( - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/iis" - "github.com/pkg/errors" -) - -const metricsetName= "website" - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("iis", metricsetName, New) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - mb.BaseMetricSet - reader *iis.Reader -} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Experimental("The iis website metricset is experimental.") - - reader, err := iis.NewReader(iis.GetPerfCounters(metricsetName)) - if err != nil { - return nil, errors.Wrap(err, "initialization of reader failed") - } - - return &MetricSet{ - BaseMetricSet: base, - reader: reader, - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right -// format. It publishes the event which is then forwarded to the output. In case -// of an error set the Error field of mb.Event or simply call report.Error(). -// Fetch fetches events and reports them upstream -func (m *MetricSet) Fetch(report mb.ReporterV2) error { - - // refresh performance counter list - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - // A flag is set if the second call has been executed else refresh will fail (reader.executed) - if m.reader.Executed { - err := m.reader.RefreshCounterPaths() - if err != nil { - return errors.Wrap(err, "failed retrieving counters") - } - } - events, err := m.reader.Read(metricsetName) - if err != nil { - return errors.Wrap(err, "failed reading counters") - } - - for _, event := range events { - isOpen := report.Event(event) - if !isOpen { - break - } - } - return nil -} - -// Close will be called when metricbeat is stopped, should close the query. -func (m *MetricSet) Close() error { - err := m.reader.Close() - if err != nil { - return errors.Wrap(err, "failed to close pdh query") - } - return nil -} - diff --git a/metricbeat/module/iis/website/website_test.go b/metricbeat/module/iis/website/website_test.go new file mode 100644 index 000000000000..81e02fe557b4 --- /dev/null +++ b/metricbeat/module/iis/website/website_test.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 05210599854e..5c36723bab83 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -31,7 +31,7 @@ import ( // CounterConfig for perfmon counters. type CounterConfig struct { - InstanceLabel string `config:"instance_label" validate:"required"` + InstanceLabel string `config:"instance_label"` InstanceName string `config:"instance_name"` MeasurementLabel string `config:"measurement_label" validate:"required"` Query string `config:"query" validate:"required"` @@ -45,8 +45,10 @@ type Config struct { CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` } +const metricsetName = "perfmon" + func init() { - mb.Registry.MustAddMetricSet("windows", "perfmon", New) + mb.Registry.MustAddMetricSet("windows", metricsetName, New) } type MetricSet struct { @@ -63,6 +65,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + if metricsetName == base.Name() { + err := validateCounterConfig(config.CounterConfig) + if err != nil { + return nil, err + } + } for _, value := range config.CounterConfig { form := strings.ToLower(value.Format) switch form { @@ -83,7 +91,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, reader: reader, - log: logp.NewLogger("perfmon"), + log: logp.NewLogger(metricsetName), }, nil } @@ -126,3 +134,12 @@ func (m *MetricSet) Close() error { } return nil } + +func validateCounterConfig(counters []CounterConfig) error { + for _, counter := range counters { + if counter.InstanceLabel == "" { + return errors.Errorf("no instance label has been configured for query %s", counter.Query) + } + } + return nil +} diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 436acdd5dba3..a57694cdd32f 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -176,7 +176,7 @@ func (r *Reader) Read() ([]mb.Event, error) { MetricSetFields: common.MapStr{}, Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), } - if val.Instance != "" { + if val.Instance != "" && r.instanceLabel[counterPath] != "" { //will ignore instance counter if ok, match := matchesParentProcess(val.Instance); ok { eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], match) diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml.disabled similarity index 92% rename from metricbeat/modules.d/iis.yml rename to metricbeat/modules.d/iis.yml.disabled index 1dd49c94be14..a81cba0baaa7 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml.disabled @@ -4,7 +4,6 @@ - module: iis metricsets: - webserver - - website enabled: true period: 10s diff --git a/metricbeat/modules.d/system.yml.disabled b/metricbeat/modules.d/system.yml similarity index 100% rename from metricbeat/modules.d/system.yml.disabled rename to metricbeat/modules.d/system.yml From f5cb0a7842d722936420953699a67cac67e0b1a8 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Jan 2020 17:31:11 +0100 Subject: [PATCH 06/36] work on build --- metricbeat/module/iis/performanceCounters.go | 107 --------- metricbeat/module/iis/reader.go | 210 ------------------ metricbeat/module/iis/webserver/manifest.yml | 15 +- metricbeat/module/iis/website/manifest.yml | 21 +- .../perfmon/pdh_integration_windows_test.go | 6 +- .../windows/perfmon/pdh_query_windows.go | 8 +- .../windows/perfmon/pdh_query_windows_test.go | 4 +- 7 files changed, 40 insertions(+), 331 deletions(-) delete mode 100644 metricbeat/module/iis/performanceCounters.go delete mode 100644 metricbeat/module/iis/reader.go diff --git a/metricbeat/module/iis/performanceCounters.go b/metricbeat/module/iis/performanceCounters.go deleted file mode 100644 index b0095d11037b..000000000000 --- a/metricbeat/module/iis/performanceCounters.go +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package iis - -var ( - website_counters = map[string]string{ - "bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", - "total_bytes_sent_per_sec": "\\Web Service(*)\\Total Bytes Sent", - "bytes_recv_per_sec": "\\Web Service(*)\\Bytes Received/sec", - "total_bytes_recv_per_sec": "\\Web Service(*)\\Total Bytes Received", - - //\Web Service(*)\Total Files Sent - //\Web Service(*)\Files Sent/sec - //\Web Service(*)\Total Files Received - //\Web Service(*)\Files Received/sec - //\Web Service(*)\Current Connections - //\Web Service(*)\Maximum Connections - //\Web Service(*)\Total Connection Attempts (all instances) - //\Web Service(*)\Total Get Requests - //\Web Service(*)\Get Requests/sec - //\Web Service(*)\Total Post Requests - //\Web Service(*)\Post Requests/sec - } - webserver_counters = map[string]string{ - "total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", - "total_bytes_recv_per_sec": "\\Web Service(_Total)\\Total Bytes Received", - //\Web Service(*)\Total Files Sent - //\Web Service(*)\Files Sent/sec - //\Web Service(*)\Total Files Received - //\Web Service(*)\Files Received/sec - //\Web Service(*)\Current Connections - //\Web Service(*)\Maximum Connections - //\Web Service(*)\Total Connection Attempts (all instances) - //\Web Service(*)\Total Get Requests - //\Web Service(*)\Get Requests/sec - //\Web Service(*)\Total Post Requests - //\Web Service(*)\Post Requests/sec - - //cache - //"cache": { - //"file_cache_count": "2", - //"file_cache_memory_usage": "699", - //"file_cache_hits": "18506471", - //"file_cache_misses": "46266060", - //"total_files_cached": "10", - //"output_cache_count": "0", - //"output_cache_memory_usage": "0", - //"output_cache_hits": "0", - //"output_cache_misses": "18506478", - //"uri_cache_count": "2", - //"uri_cache_hits": "18506452", - //"uri_cache_misses": "26", - //"total_uris_cached": "13" - //} - - } -) - -type PerformanceCounter struct { - InstanceLabel string - MeasurementLabel string - Path string - Format string -} - -func GetPerfCounters(metricset string) []PerformanceCounter { - var counters []PerformanceCounter - switch metricset { - case "website": - for k, v := range website_counters { - counter := PerformanceCounter{ - InstanceLabel: "name", - MeasurementLabel: k, - Path: v, - Format: "float", - } - counters = append(counters, counter) - } - case "webserver": - for k, v := range webserver_counters { - counter := PerformanceCounter{ - InstanceLabel: "", - MeasurementLabel: k, - Path: v, - Format: "float", - } - counters = append(counters, counter) - } - - } - return counters -} diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/reader.go deleted file mode 100644 index 3f74347818d9..000000000000 --- a/metricbeat/module/iis/reader.go +++ /dev/null @@ -1,210 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build windows - -package iis - -import ( - "regexp" - "strings" - - "github.com/elastic/beats/metricbeat/module/windows/perfmon" - - "github.com/pkg/errors" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" -) - -var ( - processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) -) - -// Reader will contain the config options -type Reader struct { - query perfmon.Query // PDH Query - instanceLabel map[string]string // Mapping of counter path to key used for the label (e.g. processor.name) - measurement map[string]string // Mapping of counter path to key used for the value (e.g. processor.cpu_time). - Executed bool // Indicates if the query has been executed. - log *logp.Logger // - counters []PerformanceCounter -} - -// NewReader creates a new instance of Reader. -func NewReader(counters []PerformanceCounter) (*Reader, error) { - var query perfmon.Query - if err := query.Open(); err != nil { - return nil, err - } - r := &Reader{ - query: query, - instanceLabel: map[string]string{}, - measurement: map[string]string{}, - log: logp.NewLogger("iis"), - counters: counters, - } - for _, counter := range counters { - childQueries, err := query.GetCounterPaths(counter.Path) - if err != nil { - if err == perfmon.PDH_CSTATUS_NO_COUNTER || err == perfmon.PDH_CSTATUS_NO_COUNTERNAME || err == perfmon.PDH_CSTATUS_NO_INSTANCE || err == perfmon.PDH_CSTATUS_NO_OBJECT { - r.log.Infow("Ignoring non existent counter", "error", err, - logp.Namespace("iis"), "query", counter.Path) - continue - } else { - query.Close() - return nil, errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Path) - } - } - // check if the pdhexpandcounterpath/pdhexpandwildcardpath functions have expanded the counter successfully. - if len(childQueries) == 0 || (len(childQueries) == 1 && strings.Contains(childQueries[0], "*")) { - // covering cases when PdhExpandWildCardPathW returns no counter paths or is unable to expand and the ignore_non_existent_counters flag is set - r.log.Infow("Ignoring non existent counter", "initial query", counter.Path, - logp.Namespace("perfmon"), "expanded query", childQueries) - continue - } - for _, v := range childQueries { - if err := query.AddCounter(v, counter.InstanceLabel, counter.Format, len(childQueries) > 1); err != nil { - return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Path) - } - r.instanceLabel[v] = counter.InstanceLabel - r.measurement[v] = counter.MeasurementLabel - } - } - - return r, nil -} - -// RefreshCounterPaths will recheck for any new instances and add them to the counter list -func (r *Reader) RefreshCounterPaths() error { - var newCounters []string - for _, counter := range r.counters { - childQueries, err := r.query.GetCounterPaths(counter.Path) - if err != nil { - if err == perfmon.PDH_CSTATUS_NO_COUNTER || err == perfmon.PDH_CSTATUS_NO_COUNTERNAME || err == perfmon.PDH_CSTATUS_NO_INSTANCE || err == perfmon.PDH_CSTATUS_NO_OBJECT { - r.log.Infow("Ignoring non existent counter", "error", err, - logp.Namespace("iis"), "query", counter.Path) - continue - } else { - return errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Path) - } - } - newCounters = append(newCounters, childQueries...) - // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list - if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { - for _, v := range childQueries { - if err := r.query.AddCounter(v, counter.InstanceLabel, counter.Format, len(childQueries) > 1); err != nil { - return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Path) - } - r.instanceLabel[v] = counter.InstanceLabel - r.measurement[v] = counter.MeasurementLabel - } - } - } - err := r.query.RemoveUnusedCounters(newCounters) - if err != nil { - return errors.Wrap(err, "failed removing unused counter values") - } - - return nil -} - -// Read executes a query and returns those values in an event. -func (r *Reader) Read(metricsetName string) ([]mb.Event, error) { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if err := r.query.CollectData(); err != nil { - return nil, errors.Wrap(err, "failed querying counter values") - } - - // Get the values. - values, err := r.query.GetFormattedCounterValues() - if err != nil { - return nil, errors.Wrap(err, "failed formatting counter values") - } - - eventMap := make(map[string]*mb.Event) - - for path, counterValues := range values { - for _, counterValue := range counterValues { - if counterValue.Err != nil && !r.Executed { - r.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", counterValue.Err, logp.Namespace("iis"), "query", path) - continue - } - if counterValue.Err != nil { - r.log.Errorw("Error while retrieving counter values", - "error", counterValue.Err, logp.Namespace("perfmon"), "query", path) - continue - } - var eventKey string - if metricsetName == "webserver" { - // Send measurements with the same instance label as part of the same event - eventKey = metricsetName - } else { - // Send every measurement as an individual event - // If a counter contains an error, it will always be sent as an individual event - eventKey = counterValue.Instance - } - - // Create a new event if the key doesn't exist in the map - if _, ok := eventMap[eventKey]; !ok { - eventMap[eventKey] = &mb.Event{ - MetricSetFields: common.MapStr{}, - } - if metricsetName != "webserver" { - //will ignore instance counter - if ok, match := matchesParentProcess(counterValue.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[path], match) - } else { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[path], counterValue.Instance) - } - } - } - event := eventMap[eventKey] - if counterValue.Measurement != nil { - event.MetricSetFields.Put(r.measurement[path], counterValue.Measurement) - } else { - event.MetricSetFields.Put(r.measurement[path], 0) - } - } - } - - // Write the values into the map. - events := make([]mb.Event, 0, len(eventMap)) - for _, val := range eventMap { - events = append(events, *val) - } - - r.Executed = true - return events, nil -} - -// Close will close the PDH query for now. -func (r *Reader) Close() error { - return r.query.Close() -} - -// matchParentProcess will try to get the parent process name -func matchesParentProcess(instanceName string) (bool, string) { - matches := processRegexp.FindStringSubmatch(instanceName) - if len(matches) == 2 { - return true, matches[1] - } - return false, instanceName -} diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 954ce281568c..a3408183834e 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -6,7 +6,20 @@ input: perfmon.ignore_non_existent_counters: true perfmon.counters: - instance_label: '' - measurement_label: network.bytes_recv_per_sec + measurement_label: network.total_bytes_recv_per_sec query: '\Web Service(_Total)\Total Bytes Received' +#"total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", +#"total_bytes_recv_per_sec": "\\Web Service(_Total)\\Total Bytes Received", +#\Web Service(*)\Total Files Sent +#\Web Service(*)\Files Sent/sec +#\Web Service(*)\Total Files Received +#\Web Service(*)\Files Received/sec +#\Web Service(*)\Current Connections +#\Web Service(*)\Maximum Connections +#\Web Service(*)\Total Connection Attempts (all instances) +#\Web Service(*)\Total Get Requests +#\Web Service(*)\Get Requests/sec +#\Web Service(*)\Total Post Requests +#\Web Service(*)\Post Requests/sec diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index 6c7a5249df1b..0bb03c71be7b 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -4,8 +4,21 @@ input: metricset: perfmon defaults: perfmon.counters: - - instance_label: physical_disk.name - measurement_label: total_bytes_recv_per_sec - query: '\Web Service(_Total)\Total Bytes Received' - + - instance_label: '' + measurement_label: bytes_recv_per_sec + query: '\Web Service(*)\Bytes Sent/sec' +#"total_bytes_sent_per_sec": "\\Web Service(*)\\Total Bytes Sent", +#"bytes_recv_per_sec": "\\Web Service(*)\\Bytes Received/sec", +#"total_bytes_recv_per_sec": "\\Web Service(*)\\Total Bytes Received", +#\Web Service(*)\Total Files Sent +#\Web Service(*)\Files Sent/sec +#\Web Service(*)\Total Files Received +#\Web Service(*)\Files Received/sec +#\Web Service(*)\Current Connections +#\Web Service(*)\Maximum Connections +#\Web Service(*)\Total Connection Attempts (all instances) +#\Web Service(*)\Total Get Requests +#\Web Service(*)\Get Requests/sec +#\Web Service(*)\Total Post Requests +#\Web Service(*)\Post Requests/sec diff --git a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go index 92bc5ca712cf..1fe12ab20d1f 100644 --- a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go +++ b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go @@ -117,7 +117,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - err = q.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = q.AddCounter(path[0], counter, false) if err != nil { t.Fatal(err) } @@ -243,7 +243,7 @@ func TestLongOutputFormat(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = query.AddCounter(path[0], counter, false) if err != nil && err != PDH_NO_MORE_DATA { t.Fatal(err) } @@ -283,7 +283,7 @@ func TestFloatOutputFormat(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = query.AddCounter(path[0], counter, false) if err != nil && err != PDH_NO_MORE_DATA { t.Fatal(err) } diff --git a/metricbeat/module/windows/perfmon/pdh_query_windows.go b/metricbeat/module/windows/perfmon/pdh_query_windows.go index 6b738285a1d0..5c5a7149d3ba 100644 --- a/metricbeat/module/windows/perfmon/pdh_query_windows.go +++ b/metricbeat/module/windows/perfmon/pdh_query_windows.go @@ -72,20 +72,20 @@ func (q *Query) AddEnglishCounter(counterPath string) (PdhCounterHandle, error) } // AddCounter adds the specified counter to the query. -func (q *Query) AddCounter(counterPath string, counterInstance string, counterFormat string, wildcard bool) error { +func (q *Query) AddCounter(counterPath string, counter CounterConfig, wildcard bool) error { if _, found := q.counters[counterPath]; found { return nil } var err error var instanceName string // Extract the instance name from the counterPath. - if counterInstance == "" || wildcard { + if counter.InstanceName == "" || wildcard { instanceName, err = matchInstanceName(counterPath) if err != nil { return err } } else { - instanceName = counterInstance + instanceName = counter.InstanceName } h, err := PdhAddCounter(q.handle, counterPath, 0) if err != nil { @@ -95,7 +95,7 @@ func (q *Query) AddCounter(counterPath string, counterInstance string, counterFo q.counters[counterPath] = &Counter{ handle: h, instanceName: instanceName, - format: getPDHFormat(counterFormat), + format: getPDHFormat(counter.Format), } return nil } diff --git a/metricbeat/module/windows/perfmon/pdh_query_windows_test.go b/metricbeat/module/windows/perfmon/pdh_query_windows_test.go index 66cc631b8851..4e45ec827180 100644 --- a/metricbeat/module/windows/perfmon/pdh_query_windows_test.go +++ b/metricbeat/module/windows/perfmon/pdh_query_windows_test.go @@ -39,7 +39,7 @@ func TestAddCounterInvalidArgWhenQueryClosed(t *testing.T) { // if windows os language is ENG then err will be nil, else the GetCounterPaths will execute the AddCounter if assert.NoError(t, err) { counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"} - err = q.AddCounter(queryPath[0], counter.InstanceName, counter.Format, false) + err = q.AddCounter(queryPath[0], counter, false) assert.Error(t, err, PDH_INVALID_HANDLE) } else { assert.Error(t, err, PDH_INVALID_ARGUMENT) @@ -75,7 +75,7 @@ func TestSuccessfulQuery(t *testing.T) { if err != nil { t.Fatal(err) } - err = q.AddCounter(queryPath[0], counter.InstanceName, counter.Format, false) + err = q.AddCounter(queryPath[0], counter, false) if err != nil { t.Fatal(err) } From c320a53e3dc89b718c6f842ab28df352221a034c Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Jan 2020 17:34:52 +0100 Subject: [PATCH 07/36] work on build --- metricbeat/module/windows/perfmon/reader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index a57694cdd32f..ea3ed3fcd5e4 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -85,7 +85,7 @@ func NewReader(config Config) (*Reader, error) { return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.Query) } for _, v := range childQueries { - if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { + if err := query.AddCounter(v, counter, len(childQueries) > 1); err != nil { return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) } r.instanceLabel[v] = counter.InstanceLabel @@ -118,7 +118,7 @@ func (r *Reader) RefreshCounterPaths() error { // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { - if err := r.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { + if err := r.query.AddCounter(v, counter, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } r.instanceLabel[v] = counter.InstanceLabel From 9d5df8255e96ae2d938dd7c03d052da3a6e69fab Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Jan 2020 17:57:22 +0100 Subject: [PATCH 08/36] build --- metricbeat/docs/modules/system.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 2 +- metricbeat/module/system/_meta/config.reference.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 02802cceccef..5a9da32636d5 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -159,7 +159,7 @@ metricbeat.modules: - module: system metricsets: - cpu # CPU usage - #- load # CPU load averages + - load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 5184c4d952d9..9fa60da81bf3 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -60,7 +60,7 @@ metricbeat.modules: - module: system metricsets: - cpu # CPU usage - #- load # CPU load averages + - load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index d851bffdcafb..c22fc6ac2794 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -1,7 +1,7 @@ - module: system metricsets: - cpu # CPU usage - #- load # CPU load averages + - load # CPU load averages - memory # Memory usage - network # Network IO - process # Per process metrics From 222648e7d5a35dc2167f75775a16a4472ebf495f Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Jan 2020 18:14:27 +0100 Subject: [PATCH 09/36] fmt update --- x-pack/metricbeat/metricbeat.reference.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index d9f5c41f435a..71ac0898adff 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -503,6 +503,14 @@ metricbeat.modules: # fields: # added to the the response in root. overwrites existing fields # key: "value" +#--------------------------------- Iis Module --------------------------------- +- module: iis + metricsets: + - webserver + enabled: true + period: 10s + + #------------------------------- Jolokia Module ------------------------------- - module: jolokia #metricsets: ["jmx"] From 00e1a087d6a9ec6370e918c1dab6e9a57809b05f Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 16 Jan 2020 13:50:07 +0100 Subject: [PATCH 10/36] temp --- metricbeat/module/iis/_meta/config.yml | 2 + metricbeat/module/iis/module.yml | 1 - metricbeat/module/iis/webserver/manifest.yml | 2 +- metricbeat/module/iis/website/manifest.yml | 24 ------- metricbeat/module/iis/website/website.go | 62 +++++++++++++++++++ .../modules.d/{iis.yml.disabled => iis.yml} | 0 .../{system.yml => system.yml.disabled} | 0 7 files changed, 65 insertions(+), 26 deletions(-) delete mode 100644 metricbeat/module/iis/website/manifest.yml create mode 100644 metricbeat/module/iis/website/website.go rename metricbeat/modules.d/{iis.yml.disabled => iis.yml} (100%) rename metricbeat/modules.d/{system.yml => system.yml.disabled} (100%) diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml index 43931eccf73d..3ff1e68e9d3c 100644 --- a/metricbeat/module/iis/_meta/config.yml +++ b/metricbeat/module/iis/_meta/config.yml @@ -1,6 +1,8 @@ - module: iis metricsets: - webserver + - website + hosts: ["local.test"] enabled: true period: 10s diff --git a/metricbeat/module/iis/module.yml b/metricbeat/module/iis/module.yml index 48b01480e3dd..5b0be61e8419 100644 --- a/metricbeat/module/iis/module.yml +++ b/metricbeat/module/iis/module.yml @@ -1,4 +1,3 @@ name: iis metricsets: - webserver -- website diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index a3408183834e..0c70f7a77727 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -7,7 +7,7 @@ input: perfmon.counters: - instance_label: '' measurement_label: network.total_bytes_recv_per_sec - query: '\Web Service(_Total)\Total Bytes Received' + query: '\Web Service(*)\Total Bytes Received' #"total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml deleted file mode 100644 index 0bb03c71be7b..000000000000 --- a/metricbeat/module/iis/website/manifest.yml +++ /dev/null @@ -1,24 +0,0 @@ -default: true -input: - module: windows - metricset: perfmon - defaults: - perfmon.counters: - - instance_label: '' - measurement_label: bytes_recv_per_sec - query: '\Web Service(*)\Bytes Sent/sec' - -#"total_bytes_sent_per_sec": "\\Web Service(*)\\Total Bytes Sent", -#"bytes_recv_per_sec": "\\Web Service(*)\\Bytes Received/sec", -#"total_bytes_recv_per_sec": "\\Web Service(*)\\Total Bytes Received", -#\Web Service(*)\Total Files Sent -#\Web Service(*)\Files Sent/sec -#\Web Service(*)\Total Files Received -#\Web Service(*)\Files Received/sec -#\Web Service(*)\Current Connections -#\Web Service(*)\Maximum Connections -#\Web Service(*)\Total Connection Attempts (all instances) -#\Web Service(*)\Total Get Requests -#\Web Service(*)\Get Requests/sec -#\Web Service(*)\Total Post Requests -#\Web Service(*)\Post Requests/sec diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go new file mode 100644 index 000000000000..915666da4139 --- /dev/null +++ b/metricbeat/module/iis/website/website.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package website + +import ( + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" +) + +// Config for the windows perfmon metricset. +type Config struct { + Hosts []string `config:"hosts"` +} + +// init registers the partition MetricSet with the central registry. +func init() { + mb.Registry.MustAddMetricSet("iis", "website", New) +} + +// MetricSet type defines all fields of the partition MetricSet +type MetricSet struct { + mb.BaseMetricSet + log *logp.Logger +} + +// New creates a new instance of the website MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The perfmon metricset is beta") + + var config Config + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + log: logp.NewLogger("website"), + }, nil +} + +// Fetch fetches events and reports them upstream +func (m *MetricSet) Fetch(r mb.ReporterV2) error { + return nil +} diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml similarity index 100% rename from metricbeat/modules.d/iis.yml.disabled rename to metricbeat/modules.d/iis.yml diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml.disabled similarity index 100% rename from metricbeat/modules.d/system.yml rename to metricbeat/modules.d/system.yml.disabled From 258ded3aed4626c778a7d255ac0509b194a115c9 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 17 Jan 2020 11:30:02 +0100 Subject: [PATCH 11/36] work on website --- metricbeat/docs/modules/iis.asciidoc | 2 + metricbeat/docs/modules/iis/website.asciidoc | 1 - metricbeat/include/list_common.go | 1 + metricbeat/metricbeat.reference.yml | 2 + metricbeat/module/iis/website/counters.go | 23 +++ metricbeat/module/iis/website/reader.go | 139 ++++++++++++++++++ metricbeat/module/iis/website/website.go | 11 +- .../windows/perfmon/defs_pdh_windows.go | 17 --- .../windows/perfmon/defs_pdh_windows_386.go | 17 --- .../windows/perfmon/defs_pdh_windows_amd64.go | 17 --- .../windows/perfmon/pdh_query_windows.go | 20 ++- .../module/windows/perfmon/pdh_windows.go | 45 ++++++ metricbeat/module/windows/perfmon/reader.go | 4 +- .../module/windows/perfmon/zpdh_windows.go | 26 ++-- metricbeat/modules.d/iis.yml | 4 +- 15 files changed, 250 insertions(+), 79 deletions(-) create mode 100644 metricbeat/module/iis/website/counters.go create mode 100644 metricbeat/module/iis/website/reader.go diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 78fad39cb021..8038fe6b8a77 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -23,6 +23,8 @@ metricbeat.modules: - module: iis metricsets: - webserver + - website + hosts: ["local.test"] enabled: true period: 10s diff --git a/metricbeat/docs/modules/iis/website.asciidoc b/metricbeat/docs/modules/iis/website.asciidoc index 1132d876ad49..ccbf0416e6f3 100644 --- a/metricbeat/docs/modules/iis/website.asciidoc +++ b/metricbeat/docs/modules/iis/website.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/iis/website/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 23278b9e3913..2df120dd089f 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -77,6 +77,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" _ "github.com/elastic/beats/metricbeat/module/iis" + _ "github.com/elastic/beats/metricbeat/module/iis/website" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9fa60da81bf3..aa887a14c6c2 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -346,6 +346,8 @@ metricbeat.modules: - module: iis metricsets: - webserver + - website + hosts: ["local.test"] enabled: true period: 10s diff --git a/metricbeat/module/iis/website/counters.go b/metricbeat/module/iis/website/counters.go new file mode 100644 index 000000000000..b570c9af3488 --- /dev/null +++ b/metricbeat/module/iis/website/counters.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website + +var webserverCounters = map[string]string{ + "network.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", + "network.bytes_received_per_sec": "\\Web Service(*)\\Bytes Received/sec", +} diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go new file mode 100644 index 000000000000..f82e05e907de --- /dev/null +++ b/metricbeat/module/iis/website/reader.go @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package website + +import ( + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "github.com/pkg/errors" + "strings" +) + +// Reader will contain the config options +type Reader struct { + query perfmon.Query // PDH Query + hosts []string // Mapping of counter path to key used for the label (e.g. processor.name) + log *logp.Logger // logger + hasRun bool // will check if the reader has run a first time +} + +// NewReader creates a new instance of Reader. +func NewReader(config Config) (*Reader, error) { + var query perfmon.Query + if err := query.Open(); err != nil { + return nil, err + } + reader := &Reader{ + query: query, + log: logp.NewLogger("perfmon"), + } + if err := reader.InitCounters(config.Hosts); err != nil { + return nil, err + } + return reader, nil +} + +func (this *Reader) InitCounters(hosts []string) error { + counters, instances, err := this.query.GetCountersAndInstances("Web Service") + _ = counters + if err != nil { + this.query.Close() + return err + } + this.hosts = filterOnInstances(hosts, instances) + var newQueries []string + for _, instance := range instances { + for _, value := range webserverCounters { + value = strings.Replace(value, "*", instance, 1) + if err := this.query.AddCounter(value, "", "float", true); err != nil { + return errors.Wrapf(err, `failed to add counter (query="%v")`, value) + } + newQueries = append(newQueries, value) + } + } + err = this.query.RemoveUnusedCounters(newQueries) + if err != nil { + return errors.Wrap(err, "failed removing unused counter values") + } + return nil +} + +// Read executes a query and returns those values in an event. +func (this *Reader) Fetch() error { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if err := this.query.CollectData(); err != nil { + return errors.Wrap(err, "failed querying counter values") + } + + // Get the values. + values, err := this.query.GetFormattedCounterValues() + if err != nil { + return errors.Wrap(err, "failed formatting counter values") + } + + for counterPath, values := range values { + for _, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + + if val.Err != nil && !this.hasRun { + this.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + + } + } + this.hasRun = true + return nil +} + +// Close will close the PDH query for now. +func (this *Reader) Close() error { + return this.query.Close() +} + +func filterOnInstances(hosts []string, instances []string) []string { + var filtered []string + // remove _Total and empty instances + for _, instance := range instances { + if instance == "_Total" || instance == "" { + continue + } + if containsHost(instance, hosts) { + filtered = append(filtered, instance) + } + } + return filtered +} + +func containsHost(item string, array []string) bool { + // if no hosts specified all instances are selected + if len(array) == 0 { + return true + } + for _, i := range array { + if i == item { + return true + } + } + return false +} diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index 915666da4139..a1f1438669b5 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -38,21 +38,26 @@ func init() { // MetricSet type defines all fields of the partition MetricSet type MetricSet struct { mb.BaseMetricSet - log *logp.Logger + log *logp.Logger + reader *Reader } // New creates a new instance of the website MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The perfmon metricset is beta") + cfgwarn.Beta("The website metricset is beta") var config Config if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - + reader, err := NewReader(config) + if err != nil { + return nil, err + } return &MetricSet{ BaseMetricSet: base, log: logp.NewLogger("website"), + reader: reader, }, nil } diff --git a/metricbeat/module/windows/perfmon/defs_pdh_windows.go b/metricbeat/module/windows/perfmon/defs_pdh_windows.go index 97f070a600a8..b415cdea64de 100644 --- a/metricbeat/module/windows/perfmon/defs_pdh_windows.go +++ b/metricbeat/module/windows/perfmon/defs_pdh_windows.go @@ -1,20 +1,3 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - // go run mkpdh_defs.go // MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT diff --git a/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go b/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go index 3995b8b9b011..95e51e1da7d4 100644 --- a/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go +++ b/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go @@ -1,20 +1,3 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo.exe -godefs defs_pdh_windows.go diff --git a/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go b/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go index 3995b8b9b011..95e51e1da7d4 100644 --- a/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go +++ b/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go @@ -1,20 +1,3 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo.exe -godefs defs_pdh_windows.go diff --git a/metricbeat/module/windows/perfmon/pdh_query_windows.go b/metricbeat/module/windows/perfmon/pdh_query_windows.go index 5c5a7149d3ba..91fb505f250d 100644 --- a/metricbeat/module/windows/perfmon/pdh_query_windows.go +++ b/metricbeat/module/windows/perfmon/pdh_query_windows.go @@ -72,20 +72,20 @@ func (q *Query) AddEnglishCounter(counterPath string) (PdhCounterHandle, error) } // AddCounter adds the specified counter to the query. -func (q *Query) AddCounter(counterPath string, counter CounterConfig, wildcard bool) error { +func (q *Query) AddCounter(counterPath string, instance string, format string, wildcard bool) error { if _, found := q.counters[counterPath]; found { return nil } var err error var instanceName string // Extract the instance name from the counterPath. - if counter.InstanceName == "" || wildcard { + if instance == "" || wildcard { instanceName, err = matchInstanceName(counterPath) if err != nil { return err } } else { - instanceName = counter.InstanceName + instanceName = instance } h, err := PdhAddCounter(q.handle, counterPath, 0) if err != nil { @@ -95,7 +95,7 @@ func (q *Query) AddCounter(counterPath string, counter CounterConfig, wildcard b q.counters[counterPath] = &Counter{ handle: h, instanceName: instanceName, - format: getPDHFormat(counter.Format), + format: getPDHFormat(format), } return nil } @@ -178,6 +178,18 @@ func (q *Query) GetFormattedCounterValues() (map[string][]CounterValue, error) { return rtn, nil } +// GetCountersAndInstances returns a list of counters and instances for a given object +func (q *Query) GetCountersAndInstances(objectName string) ([]string, []string, error) { + counters, instances, err := PdhEnumObjectItems(objectName) + if err != nil { + return nil, nil, errors.Wrapf(err, "Unable to retrieve counter and instance list for %s", objectName) + } + if len(counters) == 0 && len(instances) == 0 { + return nil, nil, errors.Errorf("Unable to retrieve counter and instance list for %s", objectName) + } + return UTF16ToStringArray(counters), UTF16ToStringArray(instances), nil +} + // ExpandWildCardPath examines local computer and returns those counter paths that match the given counter path which contains wildcard characters. func (q *Query) ExpandWildCardPath(wildCardPath string) ([]string, error) { if wildCardPath == "" { diff --git a/metricbeat/module/windows/perfmon/pdh_windows.go b/metricbeat/module/windows/perfmon/pdh_windows.go index b817ddaec161..d6d01063d8f4 100644 --- a/metricbeat/module/windows/perfmon/pdh_windows.go +++ b/metricbeat/module/windows/perfmon/pdh_windows.go @@ -41,6 +41,7 @@ import ( //sys _PdhExpandWildCardPath(dataSource *uint16, wildcardPath *uint16, expandedPathList *uint16, pathListLength *uint32) (errcode error) [failretval!=0] = pdh.PdhExpandWildCardPathW //sys _PdhExpandCounterPath(wildcardPath *uint16, expandedPathList *uint16, pathListLength *uint32) (errcode error) [failretval!=0] = pdh.PdhExpandCounterPathW //sys _PdhGetCounterInfo(counter PdhCounterHandle, text uint16, size *uint32, lpBuffer *byte) (errcode error) [failretval!=0] = pdh.PdhGetCounterInfoW +//sys _PdhEnumObjectItems(dataSource uint16, machineName uint16, objectName *uint16, counterList *uint16, counterListSize *uint32, instanceList *uint16, instanceListSize *uint32, detailLevel uint32, flags uint32) (errcode error) [failretval!=0] = pdh.PdhEnumObjectItemsW type PdhQueryHandle uintptr @@ -50,6 +51,8 @@ type PdhCounterHandle uintptr var InvalidCounterHandle = ^PdhCounterHandle(0) +const PERF_DETAIL_WIZARD = 400 + // PdhCounterInfo struct contains the performance counter details type PdhCounterInfo struct { DwLength uint32 @@ -247,6 +250,48 @@ func PdhCloseQuery(query PdhQueryHandle) error { return nil } +// PdhEnumObjectItems returns the counters and instance info for given object +func PdhEnumObjectItems(objectName string) ([]uint16, []uint16, error) { + var ( + cBuff = make([]uint16, 1) + cBuffSize = uint32(0) + iBuff = make([]uint16, 1) + iBuffSize = uint32(0) + ) + obj := windows.StringToUTF16Ptr(objectName) + if err := _PdhEnumObjectItems( + 0, + 0, + obj, + &cBuff[0], + &cBuffSize, + &iBuff[0], + &iBuffSize, + PERF_DETAIL_WIZARD, + 0); err != nil { + if PdhErrno(err.(syscall.Errno)) != PDH_MORE_DATA { + return nil, nil, PdhErrno(err.(syscall.Errno)) + } + cBuff = make([]uint16, cBuffSize) + iBuff = make([]uint16, iBuffSize) + + if err = _PdhEnumObjectItems( + 0, + 0, + obj, + &cBuff[0], + &cBuffSize, + &iBuff[0], + &iBuffSize, + PERF_DETAIL_WIZARD, + 0); err != nil { + return nil, nil, err + } + return cBuff, iBuff, nil + } + return nil, nil, nil +} + // Error returns a more explicit error message. func (e PdhErrno) Error() string { // If the value is not one of the known PDH errors then assume its a diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index ea3ed3fcd5e4..a57694cdd32f 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -85,7 +85,7 @@ func NewReader(config Config) (*Reader, error) { return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.Query) } for _, v := range childQueries { - if err := query.AddCounter(v, counter, len(childQueries) > 1); err != nil { + if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) } r.instanceLabel[v] = counter.InstanceLabel @@ -118,7 +118,7 @@ func (r *Reader) RefreshCounterPaths() error { // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { - if err := r.query.AddCounter(v, counter, len(childQueries) > 1); err != nil { + if err := r.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } r.instanceLabel[v] = counter.InstanceLabel diff --git a/metricbeat/module/windows/perfmon/zpdh_windows.go b/metricbeat/module/windows/perfmon/zpdh_windows.go index 85cb93dcc758..69d86b6da799 100644 --- a/metricbeat/module/windows/perfmon/zpdh_windows.go +++ b/metricbeat/module/windows/perfmon/zpdh_windows.go @@ -1,20 +1,3 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - // Code generated by 'go generate'; DO NOT EDIT. package perfmon @@ -66,6 +49,7 @@ var ( procPdhExpandWildCardPathW = modpdh.NewProc("PdhExpandWildCardPathW") procPdhExpandCounterPathW = modpdh.NewProc("PdhExpandCounterPathW") procPdhGetCounterInfoW = modpdh.NewProc("PdhGetCounterInfoW") + procPdhEnumObjectItemsW = modpdh.NewProc("PdhEnumObjectItemsW") ) func _PdhOpenQuery(dataSource *uint16, userData uintptr, query *PdhQueryHandle) (errcode error) { @@ -181,3 +165,11 @@ func _PdhGetCounterInfo(counter PdhCounterHandle, text uint16, size *uint32, lpB } return } + +func _PdhEnumObjectItems(dataSource uint16, machineName uint16, objectName *uint16, counterList *uint16, counterListSize *uint32, instanceList *uint16, instanceListSize *uint32, detailLevel uint32, flags uint32) (errcode error) { + r0, _, _ := syscall.Syscall9(procPdhEnumObjectItemsW.Addr(), 9, uintptr(dataSource), uintptr(machineName), uintptr(unsafe.Pointer(objectName)), uintptr(unsafe.Pointer(counterList)), uintptr(unsafe.Pointer(counterListSize)), uintptr(unsafe.Pointer(instanceList)), uintptr(unsafe.Pointer(instanceListSize)), uintptr(detailLevel), uintptr(flags)) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml index a81cba0baaa7..41c470cd92fc 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml @@ -4,6 +4,8 @@ - module: iis metricsets: - webserver + - website + hosts: ["local.test"] enabled: true - period: 10s + period: 30s From 5f30e5a4022236017aa5ea490a8abbfa254414a3 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 21 Jan 2020 10:42:56 +0100 Subject: [PATCH 12/36] temp --- metricbeat/module/iis/webserver/manifest.yml | 134 ++++++++++++++++--- metricbeat/module/iis/website/reader.go | 4 +- metricbeat/modules.d/iis.yml | 2 - 3 files changed, 119 insertions(+), 21 deletions(-) diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 0c70f7a77727..01ee72d92c69 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -3,23 +3,123 @@ input: module: windows metricset: perfmon defaults: + perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true perfmon.counters: + #network - instance_label: '' - measurement_label: network.total_bytes_recv_per_sec - query: '\Web Service(*)\Total Bytes Received' - - -#"total_bytes_sent_per_sec": "\\Web Service(_Total)\\Total Bytes Sent", -#"total_bytes_recv_per_sec": "\\Web Service(_Total)\\Total Bytes Received", -#\Web Service(*)\Total Files Sent -#\Web Service(*)\Files Sent/sec -#\Web Service(*)\Total Files Received -#\Web Service(*)\Files Received/sec -#\Web Service(*)\Current Connections -#\Web Service(*)\Maximum Connections -#\Web Service(*)\Total Connection Attempts (all instances) -#\Web Service(*)\Total Get Requests -#\Web Service(*)\Get Requests/sec -#\Web Service(*)\Total Post Requests -#\Web Service(*)\Post Requests/sec + measurement_label: network.total_bytes_received + query: '\Web Service(_Total)\Total Bytes Received' + - instance_label: '' + measurement_label: network.total_bytes_sent + query: '\Web Service(_Total)\Total Bytes Sent' + - instance_label: '' + measurement_label: network.bytes_sent_per_sec + query: '\Web Service(_Total)\Bytes Sent/sec' + - instance_label: '' + measurement_label: network.bytes_received_per_sec + query: '\Web Service(_Total)\Bytes Received/sec' + - instance_label: '' + measurement_label: network.current_connections + query: 'Web Service(_Total)\Current Connections' + - instance_label: '' + measurement_label: network.maximum_connections + query: '\Web Service(_Total)\Maximum Connections' + - instance_label: '' + measurement_label: network.total_connection_attempts + query: '\Web Service(_Total)\Total Connection Attempts (all instances)' + - instance_label: '' + measurement_label: network.total_get_requests + query: '\Web Service(_Total)\Total Get Requests' + - instance_label: '' + measurement_label: network.get_requests_per_sec + query: '\Web Service(_Total)\Get Requests/sec' + - instance_label: '' + measurement_label: network.total_post_requests + query: '\Web Service(_Total)\Total Post Requests' + - instance_label: '' + measurement_label: network.post_requests_per_sec + query: '\Web Service(_Total)\Post Requests/sec' + - instance_label: '' + measurement_label: network.total_delete_requests + query: '\Web Service(_Total)\Total Delete Requests' + - instance_label: '' + measurement_label: network.delete_requests_per_sec + query: '\Web Service(_Total)\Delete Requests/sec' + - instance_label: '' + measurement_label: network.service_uptime + query: '\Web Service(*)\Service Uptime' + #cache + - instance_label: '' + measurement_label: cache.current_files_cached + query: '\Web Service Cache\Current Files Cached' + - instance_label: '' + measurement_label: cache.total_files_cached + query: '\Web Service Cache\Total Files Cached' + - instance_label: '' + measurement_label: cache.file_cache_hits + query: '\Web Service Cache\File Cache Hits' + - instance_label: '' + measurement_label: cache.file_cache_mises + query: '\Web Service Cache\File Cache Misses' + - instance_label: '' + measurement_label: cache.current_file_cache_memory_usage + query: '\Web Service Cache\Current File Cache Memory Usage' + - instance_label: '' + measurement_label: cache.maximum_file_cache_memory_usage + query: '\Web Service Cache\Maximum File Cache Memory Usage' + - instance_label: '' + measurement_label: cache.current_uris_cached + query: '\Web Service Cache\Current URIs Cached' + - instance_label: '' + measurement_label: cache.total_uris_cached + query: '\Web Service Cache\Total URIs Cached' + - instance_label: '' + measurement_label: cache.uri_cache_hits + query: '\Web Service Cache\URI Cache Hits' + - instance_label: '' + measurement_label: cache.uri_cache_misses + query: '\Web Service Cache\URI Cache Misses' + - instance_label: '' + measurement_label: cache.output_cache_current_memory_usage + query: '\Web Service Cache\Output Cache Current Memory Usage' + - instance_label: '' + measurement_label: cache.output_cache_current_items + query: '\Web Service Cache\Output Cache Current Items' + - instance_label: '' + measurement_label: cache.output_cache_total_hits + query: '\Web Service Cache\Output Cache Total Hits' + - instance_label: '' + measurement_label: cache.output_cache_total_misses + query: '\Web Service Cache\Output Cache Total Misses' + #process + - instance_label: '' + measurement_label: process.cpu_usage_perc + query: '\Process(w3wp*)\% Processor Time' + - instance_label: '' + measurement_label: process.handle_count + query: '\Process(w3wp*)\Handle Count' + - instance_label: '' + measurement_label: process.thread_count + query: '\Process(w3wp*)\Thread Count' + - instance_label: '' + measurement_label: process.working_set + query: '\Process(w3wp*)\Working Set' + - instance_label: '' + measurement_label: process.private_byte + query: '\Process(w3wp*)\Private Bytes' + - instance_label: '' + measurement_label: process.virtual_bytes + query: '\Process(w3wp*)\Virtual Bytes' + - instance_label: '' + measurement_label: process.page_faults_per_Sec + query: '\Process(w3wp*)\Page Faults/sec' + - instance_label: '' + measurement_label: process.io_read_operations_per_sec + query: '\Process(w3wp*)\IO Read Operations/sec' + - instance_label: '' + measurement_label: process.io_write_operations_per_sec + query: '\Process(w3wp*)\IO Write Operations/sec' + - instance_label: '' + measurement_label: process.virtual_bytes + query: '\Process(w3wp*)\Virtual Bytes' diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go index f82e05e907de..3fdbfbd7a2e6 100644 --- a/metricbeat/module/iis/website/reader.go +++ b/metricbeat/module/iis/website/reader.go @@ -42,7 +42,7 @@ func NewReader(config Config) (*Reader, error) { } reader := &Reader{ query: query, - log: logp.NewLogger("perfmon"), + log: logp.NewLogger("website"), } if err := reader.InitCounters(config.Hosts); err != nil { return nil, err @@ -59,7 +59,7 @@ func (this *Reader) InitCounters(hosts []string) error { } this.hosts = filterOnInstances(hosts, instances) var newQueries []string - for _, instance := range instances { + for _, instance := range this.hosts { for _, value := range webserverCounters { value = strings.Replace(value, "*", instance, 1) if err := this.query.AddCounter(value, "", "float", true); err != nil { diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml index 41c470cd92fc..5593126a2501 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml @@ -4,8 +4,6 @@ - module: iis metricsets: - webserver - - website - hosts: ["local.test"] enabled: true period: 30s From 92e5873433b2b62c4fedd62bcc334b100aec2390 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 22 Jan 2020 10:40:02 +0100 Subject: [PATCH 13/36] temp --- metricbeat/module/iis/webserver/manifest.yml | 4 +-- metricbeat/module/iis/website/reader.go | 6 ++--- metricbeat/module/windows/perfmon/perfmon.go | 26 ++++++++++++++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 01ee72d92c69..fdf9a24ae451 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -5,6 +5,7 @@ input: defaults: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true + perfmon.group_all_counter: true perfmon.counters: #network - instance_label: '' @@ -120,6 +121,3 @@ input: - instance_label: '' measurement_label: process.io_write_operations_per_sec query: '\Process(w3wp*)\IO Write Operations/sec' - - instance_label: '' - measurement_label: process.virtual_bytes - query: '\Process(w3wp*)\Virtual Bytes' diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go index 3fdbfbd7a2e6..e87d87c49067 100644 --- a/metricbeat/module/iis/website/reader.go +++ b/metricbeat/module/iis/website/reader.go @@ -21,14 +21,14 @@ package website import ( "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/module/windows/perfmon" + "github.com/elastic/beats/metricbeat/helper/windows/pdh" "github.com/pkg/errors" "strings" ) // Reader will contain the config options type Reader struct { - query perfmon.Query // PDH Query + query pdh.Query // PDH Query hosts []string // Mapping of counter path to key used for the label (e.g. processor.name) log *logp.Logger // logger hasRun bool // will check if the reader has run a first time @@ -36,7 +36,7 @@ type Reader struct { // NewReader creates a new instance of Reader. func NewReader(config Config) (*Reader, error) { - var query perfmon.Query + var query pdh.Query if err := query.Open(); err != nil { return nil, err } diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 75c5ae0d4606..962ac4fc102a 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -20,6 +20,7 @@ package perfmon import ( + "github.com/elastic/beats/libbeat/common" "strings" "github.com/pkg/errors" @@ -43,6 +44,7 @@ type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` + GroupAllCounters bool `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment } const metricsetName = "perfmon" @@ -116,16 +118,30 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "failed reading counters") } - - for _, event := range events { - isOpen := report.Event(event) - if !isOpen { - break + if m.reader.config.GroupAllCounters { + event := groupAllEvents(events) + report.Event(event) + } else { + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } } } return nil } +func groupAllEvents(events []mb.Event) mb.Event { + ev := mb.Event{ + MetricSetFields: common.MapStr{}, + } + for _, event := range events { + ev.MetricSetFields.Update(event.MetricSetFields) + } + return ev +} + // Close will be called when metricbeat is stopped, should close the query. func (m *MetricSet) Close() error { err := m.reader.Close() From f18b2982e82e0fa9378b1f96fc4e39102f1a8895 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 23 Jan 2020 15:38:24 +0100 Subject: [PATCH 14/36] manifest changes --- metricbeat/module/iis/webserver/manifest.yml | 2 +- metricbeat/module/windows/perfmon/reader.go | 99 ++++++++++++++++---- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index fdf9a24ae451..282b080bfb37 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -49,7 +49,7 @@ input: query: '\Web Service(_Total)\Delete Requests/sec' - instance_label: '' measurement_label: network.service_uptime - query: '\Web Service(*)\Service Uptime' + query: '\Web Service(_Total)\Service Uptime' #cache - instance_label: '' measurement_label: cache.current_files_cached diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 40e59499ccdd..b1932ad0bdf9 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -137,33 +137,46 @@ func (r *Reader) RefreshCounterPaths() error { } // Read executes a query and returns those values in an event. -func (r *Reader) Read() ([]mb.Event, error) { +func (this *Reader) Read() ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if err := r.query.CollectData(); err != nil { + if err := this.query.CollectData(); err != nil { return nil, errors.Wrap(err, "failed querying counter values") } // Get the values. - values, err := r.query.GetFormattedCounterValues() + values, err := this.query.GetFormattedCounterValues() if err != nil { return nil, errors.Wrap(err, "failed formatting counter values") } + var events []mb.Event + if this.config.GroupAllCounters { + if event := this.groupToEvent(values); event.MetricSetFields != nil { + events = append(events, event) + } + }else { + if grouped := this.groupToEvents(values); len(grouped)>0 { + events = append(events, grouped...) + } + } - eventMap := make(map[string]*mb.Event) + this.executed = true + return events, nil +} - for counterPath, values := range values { +func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { + eventMap := make(map[string]*mb.Event) + for counterPath, values := range counters { for ind, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !r.executed { - r.log.Debugw("Ignoring the first measurement because the data isn't ready", + if val.Err != nil && !this.executed { + this.log.Debugw("Ignoring the first measurement because the data isn't ready", "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } - var eventKey string - if r.config.GroupMeasurements && val.Err == nil { + if this.config.GroupMeasurements && val.Err == nil { // Send measurements with the same instance label as part of the same event eventKey = val.Instance } else { @@ -178,20 +191,20 @@ func (r *Reader) Read() ([]mb.Event, error) { MetricSetFields: common.MapStr{}, Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), } - if val.Instance != "" && r.instanceLabel[counterPath] != "" { + if val.Instance != "" && this.instanceLabel[counterPath] != "" { //will ignore instance counter if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], match) + eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], match) } else { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], val.Instance) + eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], val.Instance) } } } event := eventMap[eventKey] if val.Measurement != nil { - event.MetricSetFields.Put(r.measurement[counterPath], val.Measurement) + event.MetricSetFields.Put(this.measurement[counterPath], val.Measurement) } else { - event.MetricSetFields.Put(r.measurement[counterPath], 0) + event.MetricSetFields.Put(this.measurement[counterPath], 0) } } } @@ -201,14 +214,64 @@ func (r *Reader) Read() ([]mb.Event, error) { for _, val := range eventMap { events = append(events, *val) } + return events +} - r.executed = true - return events, nil +func (this *Reader)groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { + event := mb.Event{ + MetricSetFields: common.MapStr{}, + } + grouped:= make(map[string][]common.MapStr) + eventMap := make(map[string]*mb.Event) + for counterPath, values := range counters { + for ind, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !this.executed { + this.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + var eventKey string + if this.config.GroupMeasurements && val.Err == nil { + // Send measurements with the same instance label as part of the same event + eventKey = val.Instance + } else { + // Send every measurement as an individual event + // If a counter contains an error, it will always be sent as an individual event + eventKey = counterPath + strconv.Itoa(ind) + } + + // Create a new event if the key doesn't exist in the map + if _, ok := eventMap[eventKey]; !ok { + eventMap[eventKey] = &mb.Event{ + MetricSetFields: common.MapStr{}, + Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), + } + if val.Instance != "" && this.instanceLabel[counterPath] != "" { + //will ignore instance counter + if ok, match := matchesParentProcess(val.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], match) + } else { + eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], val.Instance) + } + } + } + event := eventMap[eventKey] + if val.Measurement != nil { + event.MetricSetFields.Put(this.measurement[counterPath], val.Measurement) + } else { + event.MetricSetFields.Put(this.measurement[counterPath], 0) + } + } + } + + return event } // Close will close the PDH query for now. -func (r *Reader) Close() error { - return r.query.Close() +func (this *Reader) Close() error { + return this.query.Close() } // matchParentProcess will try to get the parent process name From b770c575e65284a8caf9ca9d80e5c417418c8685 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 24 Jan 2020 15:51:09 +0100 Subject: [PATCH 15/36] temp --- metricbeat/module/windows/perfmon/reader.go | 24 ++++++--------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index b1932ad0bdf9..d5be7a601497 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -222,7 +222,6 @@ func (this *Reader)groupToEvent(counters map[string][]pdh.CounterValue) mb.Event MetricSetFields: common.MapStr{}, } grouped:= make(map[string][]common.MapStr) - eventMap := make(map[string]*mb.Event) for counterPath, values := range counters { for ind, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. @@ -241,27 +240,16 @@ func (this *Reader)groupToEvent(counters map[string][]pdh.CounterValue) mb.Event // If a counter contains an error, it will always be sent as an individual event eventKey = counterPath + strconv.Itoa(ind) } - // Create a new event if the key doesn't exist in the map - if _, ok := eventMap[eventKey]; !ok { - eventMap[eventKey] = &mb.Event{ - MetricSetFields: common.MapStr{}, - Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), - } - if val.Instance != "" && this.instanceLabel[counterPath] != "" { - //will ignore instance counter - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], match) - } else { - eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], val.Instance) - } - } + if _, ok := grouped[eventKey]; !ok { + grouped[eventKey] = make([]common.MapStr,0) + //[]common.MapStr {{"Error":errors.Wrapf(val.Err, "failed on query=%v", counterPath)}} } - event := eventMap[eventKey] + if val.Measurement != nil { - event.MetricSetFields.Put(this.measurement[counterPath], val.Measurement) + grouped[eventKey] = append(grouped[eventKey], common.MapStr{this.measurement[counterPath]: val.Measurement}) } else { - event.MetricSetFields.Put(this.measurement[counterPath], 0) + grouped[eventKey] = append(grouped[eventKey], common.MapStr{this.measurement[counterPath]: 0}) } } } From ea28cbef4513d18836e887f3cb1099433a91ef09 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 28 Jan 2020 14:28:46 +0100 Subject: [PATCH 16/36] temp --- metricbeat/module/iis/webserver/manifest.yml | 2 +- metricbeat/module/iis/website/counters.go | 47 ++++++++++- metricbeat/module/iis/website/reader.go | 2 + metricbeat/module/windows/perfmon/perfmon.go | 28 ++----- metricbeat/module/windows/perfmon/reader.go | 82 +++++++++---------- .../{system.yml.disabled => system.yml} | 26 +----- 6 files changed, 94 insertions(+), 93 deletions(-) rename metricbeat/modules.d/{system.yml.disabled => system.yml} (53%) diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 282b080bfb37..3ed0eb25f5bf 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -61,7 +61,7 @@ input: measurement_label: cache.file_cache_hits query: '\Web Service Cache\File Cache Hits' - instance_label: '' - measurement_label: cache.file_cache_mises + measurement_label: cache.file_cache_misses query: '\Web Service Cache\File Cache Misses' - instance_label: '' measurement_label: cache.current_file_cache_memory_usage diff --git a/metricbeat/module/iis/website/counters.go b/metricbeat/module/iis/website/counters.go index b570c9af3488..708398eb6bc8 100644 --- a/metricbeat/module/iis/website/counters.go +++ b/metricbeat/module/iis/website/counters.go @@ -17,7 +17,46 @@ package website -var webserverCounters = map[string]string{ - "network.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", - "network.bytes_received_per_sec": "\\Web Service(*)\\Bytes Received/sec", -} +var ( + webserverCounters = map[string]string{ + "network.total_bytes_received": "\\Web Service(*)\\Total Bytes Received", + "network.total_bytes_sent": "\\Web Service(*)\\Total Bytes Sent", + "network.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", + "network.bytes_received_per_sec": "\\Web Service(*)\\Bytes Received/sec", + "network.current_connections": "\\Web Service(*)\\Current Connections", + "network.maximum_connections": "\\Web Service(*)\\Maximum Connections", + "network.total_connection_attempts": "\\Web Service(*)\\Total Connection Attempts (all instances)", + "network.total_get_requests": "\\Web Service(*)\\Total Get Requests", + "network.get_requests_per_sec": "\\Web Service(*)\\Get Requests/sec", + "network.total_post_requests": "\\Web Service(*)\\Total Post Requests", + "network.post_requests_per_sec": "\\Web Service(*)\\Post Requests/sec", + "network.total_delete_requests": "\\Web Service(*)\\Total Delete Requests", + "network.delete_requests_per_sec": "\\Web Service(*)\\Delete Requests/sec", + "network.service_uptime": "\\Web Service(*)\\Service Uptime", + "cache.current_files_cached": "\\Web Service Cache\\Current Files Cached", + "cache.total_files_cached": "\\Web Service Cache\\Total Files Cached", + "cache.file_cache_hits": "\\Web Service Cache\\File Cache Hits", + "cache.file_cache_misses": "\\Web Service Cache\\File Cache Misses", + "cache.current_file_cache_memory_usage": "\\Web Service Cache\\Current File Cache Memory Usage", + "cache.maximum_file_cache_memory_usage": "\\Web Service Cache\\Maximum File Cache Memory Usage", + "cache.current_uris_cached": "\\Web Service Cache\\Current URIs Cached", + "cache.total_uris_cached": "\\Web Service Cache\\Total URIs Cached", + "cache.uri_cache_hits": "\\Web Service Cache\\URI Cache Hits", + "cache.uri_cache_misses": "\\Web Service Cache\\URI Cache Misses", + "cache.output_cache_current_memory_usage": "\\Web Service Cache\\Output Cache Current Memory Usage", + "cache.output_cache_current_items": "\\Web Service Cache\\Output Cache Current Items", + "cache.output_cache_total_hits": "\\Web Service Cache\\Output Cache Total Hits", + "cache.output_cache_total_misses": "\\Web Service Cache\\Output Cache Total Misses", + } + processCounters = map[string]string{ + "process.cpu_usage_perc": "\\Process(*)\\% Processor Time", + "process.handle_count": "\\Process(*)\\Handle Count", + "process.thread_count": "\\Process(*)\\Thread Count", + "process.working_set": "\\Process(*)\\Working Set", + "process.private_byte": "\\Process(*)\\Private Bytes", + "process.virtual_bytes": "\\Process(*)\\Virtual Bytes", + "process.page_faults_per_sec": "\\Process(*)\\Page Faults/sec", + "process.io_read_operations_per_sec": "\\Process(*)\\IO Read Operations/sec", + "process.io_write_operations_per_sec": "\\Process(*)\\IO Write Operations/sec", + } +) diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go index e87d87c49067..5be37549f150 100644 --- a/metricbeat/module/iis/website/reader.go +++ b/metricbeat/module/iis/website/reader.go @@ -68,6 +68,7 @@ func (this *Reader) InitCounters(hosts []string) error { newQueries = append(newQueries, value) } } + err = this.query.RemoveUnusedCounters(newQueries) if err != nil { return errors.Wrap(err, "failed removing unused counter values") @@ -137,3 +138,4 @@ func containsHost(item string, array []string) bool { } return false } + diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 962ac4fc102a..44443b407b73 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -20,7 +20,6 @@ package perfmon import ( - "github.com/elastic/beats/libbeat/common" "strings" "github.com/pkg/errors" @@ -114,32 +113,19 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "failed retrieving counters") } } - events, err := m.reader.Read() + events, err := m.reader.Read(m.BaseMetricSet.Name()) if err != nil { return errors.Wrap(err, "failed reading counters") } - if m.reader.config.GroupAllCounters { - event := groupAllEvents(events) - report.Event(event) - } else { - for _, event := range events { - isOpen := report.Event(event) - if !isOpen { - break - } - } - } - return nil -} -func groupAllEvents(events []mb.Event) mb.Event { - ev := mb.Event{ - MetricSetFields: common.MapStr{}, - } for _, event := range events { - ev.MetricSetFields.Update(event.MetricSetFields) + isOpen := report.Event(event) + if !isOpen { + break + } } - return ev + + return nil } // Close will be called when metricbeat is stopped, should close the query. diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index d5be7a601497..640f38e4a4ac 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -33,9 +33,9 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -var ( - processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) -) +var processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) + +const instanceCountLabel = ":count" // Reader will contain the config options type Reader struct { @@ -94,21 +94,20 @@ func NewReader(config Config) (*Reader, error) { r.measurement[v] = counter.MeasurementLabel } } - return r, nil } // RefreshCounterPaths will recheck for any new instances and add them to the counter list -func (r *Reader) RefreshCounterPaths() error { +func (this *Reader) RefreshCounterPaths() error { var newCounters []string - for _, counter := range r.config.CounterConfig { - childQueries, err := r.query.GetCounterPaths(counter.Query) + for _, counter := range this.config.CounterConfig { + childQueries, err := this.query.GetCounterPaths(counter.Query) if err != nil { - if r.config.IgnoreNECounters { + if this.config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: - r.log.Infow("Ignoring non existent counter", "error", err, + this.log.Infow("Ignoring non existent counter", "error", err, logp.Namespace("perfmon"), "query", counter.Query) continue } @@ -120,15 +119,15 @@ func (r *Reader) RefreshCounterPaths() error { // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { - if err := r.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { + if err := this.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } - r.instanceLabel[v] = counter.InstanceLabel - r.measurement[v] = counter.MeasurementLabel + this.instanceLabel[v] = counter.InstanceLabel + this.measurement[v] = counter.MeasurementLabel } } } - err := r.query.RemoveUnusedCounters(newCounters) + err := this.query.RemoveUnusedCounters(newCounters) if err != nil { return errors.Wrap(err, "failed removing unused counter values") } @@ -137,7 +136,7 @@ func (r *Reader) RefreshCounterPaths() error { } // Read executes a query and returns those values in an event. -func (this *Reader) Read() ([]mb.Event, error) { +func (this *Reader) Read(metricset string) ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if err := this.query.CollectData(); err != nil { @@ -150,22 +149,19 @@ func (this *Reader) Read() ([]mb.Event, error) { return nil, errors.Wrap(err, "failed formatting counter values") } var events []mb.Event - if this.config.GroupAllCounters { - if event := this.groupToEvent(values); event.MetricSetFields != nil { - events = append(events, event) - } - }else { - if grouped := this.groupToEvents(values); len(grouped)>0 { - events = append(events, grouped...) - } + if this.config.GroupAllCounters && (metricsetName != metricset) { + event := this.groupToEvent(values) + events = append(events, event) + } else { + events = this.groupToEvents(values) } - this.executed = true return events, nil } func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { eventMap := make(map[string]*mb.Event) + for counterPath, values := range counters { for ind, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. @@ -175,6 +171,7 @@ func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.E "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } + var eventKey string if this.config.GroupMeasurements && val.Err == nil { // Send measurements with the same instance label as part of the same event @@ -208,7 +205,6 @@ func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.E } } } - // Write the values into the map. events := make([]mb.Event, 0, len(eventMap)) for _, val := range eventMap { @@ -217,13 +213,13 @@ func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.E return events } -func (this *Reader)groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { +func (this *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { event := mb.Event{ MetricSetFields: common.MapStr{}, } - grouped:= make(map[string][]common.MapStr) + measurements := make(map[string]float64, 0) for counterPath, values := range counters { - for ind, val := range values { + for _, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if val.Err != nil && !this.executed { @@ -231,29 +227,27 @@ func (this *Reader)groupToEvent(counters map[string][]pdh.CounterValue) mb.Event "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } - var eventKey string - if this.config.GroupMeasurements && val.Err == nil { - // Send measurements with the same instance label as part of the same event - eventKey = val.Instance + + if _, ok := measurements[this.measurement[counterPath]]; !ok { + measurements[this.measurement[counterPath]] = val.Measurement.(float64) + measurements[this.measurement[counterPath]+instanceCountLabel] = 1 } else { - // Send every measurement as an individual event - // If a counter contains an error, it will always be sent as an individual event - eventKey = counterPath + strconv.Itoa(ind) - } - // Create a new event if the key doesn't exist in the map - if _, ok := grouped[eventKey]; !ok { - grouped[eventKey] = make([]common.MapStr,0) - //[]common.MapStr {{"Error":errors.Wrapf(val.Err, "failed on query=%v", counterPath)}} + measurements[this.measurement[counterPath]+instanceCountLabel] = measurements[this.measurement[counterPath]+instanceCountLabel] + 1 + measurements[this.measurement[counterPath]] = measurements[this.measurement[counterPath]] + val.Measurement.(float64) } - - if val.Measurement != nil { - grouped[eventKey] = append(grouped[eventKey], common.MapStr{this.measurement[counterPath]: val.Measurement}) + } + } + for key, val := range measurements { + if strings.Contains(key, instanceCountLabel) { + if val == 1 { + continue } else { - grouped[eventKey] = append(grouped[eventKey], common.MapStr{this.measurement[counterPath]: 0}) + event.MetricSetFields.Put(strings.Split(key, ".")[0]+".instances", val) } + } else { + event.MetricSetFields.Put(key, val) } } - return event } diff --git a/metricbeat/modules.d/system.yml.disabled b/metricbeat/modules.d/system.yml similarity index 53% rename from metricbeat/modules.d/system.yml.disabled rename to metricbeat/modules.d/system.yml index 603dee8c0a45..0b34e0e04554 100644 --- a/metricbeat/modules.d/system.yml.disabled +++ b/metricbeat/modules.d/system.yml @@ -4,9 +4,9 @@ - module: system period: 10s metricsets: - - cpu - - load - - memory + #- cpu + #- load + #- memory - network - process - process_summary @@ -19,23 +19,3 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory - -- module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' - -- module: system - period: 15m - metricsets: - - uptime - -#- module: system -# period: 5m -# metricsets: -# - raid -# raid.mount_point: '/' From 397296bd5bf9d2da6a7d5ec884b3d50023055f0f Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 31 Jan 2020 15:45:17 +0100 Subject: [PATCH 17/36] work on wp --- metricbeat/docs/fields.asciidoc | 52 +++++----- metricbeat/docs/modules_list.asciidoc | 24 ++--- metricbeat/module/iis/website/reader.go | 94 +++++++++++++----- metricbeat/module/iis/website/runner.go | 96 +++++++++++++++++++ metricbeat/module/iis/website/website.go | 34 ++++++- .../modules.d/{iis.yml => iis.yml.disabled} | 4 +- metricbeat/modules.d/system.yml | 6 +- 7 files changed, 243 insertions(+), 67 deletions(-) create mode 100644 metricbeat/module/iis/website/runner.go rename metricbeat/modules.d/{iis.yml => iis.yml.disabled} (74%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ce63ff317076..810cb9a7fccd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -42,8 +42,8 @@ grouped in the following categories: * <> * <> * <> -* <> * <> +* <> * <> * <> * <> @@ -15863,6 +15863,31 @@ IBM MQ module +[[exported-fields-iis]] +== iis fields + +iis module + + + +[float] +=== iis + + + + +[float] +=== webserver + +webserver + + +[float] +=== website + +website + + [[exported-fields-istio]] == istio fields @@ -16176,31 +16201,6 @@ type: keyword -- -[[exported-fields-iis]] -== iis fields - -iis module - - - -[float] -=== iis - - - - -[float] -=== webserver - -webserver - - -[float] -=== website - -website - - [[exported-fields-jolokia]] == Jolokia fields diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 8d5303ded5e3..6c48caef70cb 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -16,14 +16,14 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.13+| .13+| |<> beta[] +.13+| .13+| |<> beta[] |<> |<> beta[] |<> |<> |<> -|<> beta[] -|<> +|<> beta[] +|<> |<> |<> |<> beta[] @@ -101,14 +101,14 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> -|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.1+| .1+| |<> beta[] -|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> beta[] -|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.2+| .2+| |<> beta[] -|<> beta[] -|<> |image:./images/icon-no.png[No prebuilt dashboards] | +|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | +.1+| .1+| |<> beta[] +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.2+| .2+| |<> beta[] +|<> beta[] +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> beta[] +|<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .5+| .5+| |<> beta[] @@ -264,8 +264,8 @@ include::modules/graphite.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/http.asciidoc[] include::modules/ibmmq.asciidoc[] -include::modules/istio.asciidoc[] include::modules/iis.asciidoc[] +include::modules/istio.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go index 5be37549f150..d4b74df38bb6 100644 --- a/metricbeat/module/iis/website/reader.go +++ b/metricbeat/module/iis/website/reader.go @@ -20,20 +20,28 @@ package website import ( + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper/windows/pdh" + "github.com/elastic/beats/metricbeat/mb" "github.com/pkg/errors" "strings" ) // Reader will contain the config options type Reader struct { - query pdh.Query // PDH Query - hosts []string // Mapping of counter path to key used for the label (e.g. processor.name) - log *logp.Logger // logger + query pdh.Query // PDH Query + hosts []Host // Mapping of counter path to key used for the label (e.g. processor.name) + log *logp.Logger // logger hasRun bool // will check if the reader has run a first time } +type Host struct { + name string + processId int + counters map[string]string +} + // NewReader creates a new instance of Reader. func NewReader(config Config) (*Reader, error) { var query pdh.Query @@ -59,13 +67,30 @@ func (this *Reader) InitCounters(hosts []string) error { } this.hosts = filterOnInstances(hosts, instances) var newQueries []string - for _, instance := range this.hosts { - for _, value := range webserverCounters { - value = strings.Replace(value, "*", instance, 1) + for i, instance := range this.hosts { + this.hosts[i].counters = make(map[string]string) + for key, value := range webserverCounters { + value = strings.Replace(value, "*", instance.name, 1) + if err := this.query.AddCounter(value, "", "float", true); err != nil { + return errors.Wrapf(err, `failed to add counter (query="%v")`, value) + } + newQueries = append(newQueries, value) + this.hosts[i].counters[value] = key + } + var processId int + processId, err = GetProcessId(instance.name) + this.hosts[i].processId = processId + if err != nil { + this.log.Errorf("Cannot find attached worker process %s", err) + continue + } + for key, value := range processCounters { + value = strings.Replace(value, "*", string(processId), 1) if err := this.query.AddCounter(value, "", "float", true); err != nil { return errors.Wrapf(err, `failed to add counter (query="%v")`, value) } newQueries = append(newQueries, value) + this.hosts[i].counters[value] = key } } @@ -77,34 +102,48 @@ func (this *Reader) InitCounters(hosts []string) error { } // Read executes a query and returns those values in an event. -func (this *Reader) Fetch() error { +func (this *Reader) Fetch() ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if err := this.query.CollectData(); err != nil { - return errors.Wrap(err, "failed querying counter values") + return nil, errors.Wrap(err, "failed querying counter values") } // Get the values. values, err := this.query.GetFormattedCounterValues() if err != nil { - return errors.Wrap(err, "failed formatting counter values") + return nil, errors.Wrap(err, "failed formatting counter values") } - - for counterPath, values := range values { - for _, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - - if val.Err != nil && !this.hasRun { - this.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) - continue + events := make(map[string]mb.Event) + for _, host := range this.hosts { + events[host.name] = mb.Event{ + MetricSetFields: common.MapStr{ + "name": host.name, + "worker_process_id": host.processId, + }, + } + for counterPath, values := range values { + for _, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !this.hasRun { + this.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("website"), "query", counterPath) + continue + } + if val.Instance == host.name || val.Instance == string(host.processId) { + events[host.name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) + } } - } } + this.hasRun = true - return nil + results := make([]mb.Event, 0, len(events)) + for _, val := range events { + results = append(results, val) + } + return results, nil } // Close will close the PDH query for now. @@ -112,15 +151,15 @@ func (this *Reader) Close() error { return this.query.Close() } -func filterOnInstances(hosts []string, instances []string) []string { - var filtered []string +func filterOnInstances(hosts []string, instances []string) []Host { + filtered := make([]Host, 0) // remove _Total and empty instances for _, instance := range instances { if instance == "_Total" || instance == "" { continue } if containsHost(instance, hosts) { - filtered = append(filtered, instance) + filtered = append(filtered, Host{name: instance}) } } return filtered @@ -139,3 +178,10 @@ func containsHost(item string, array []string) bool { return false } +func GetProcessId(host string) (int, error) { + appPool, err := GetApplicationPool(host) + if err != nil { + return 0, err + } + return GetWorkerProcessId(appPool) +} diff --git a/metricbeat/module/iis/website/runner.go b/metricbeat/module/iis/website/runner.go new file mode 100644 index 000000000000..4fa4110ff309 --- /dev/null +++ b/metricbeat/module/iis/website/runner.go @@ -0,0 +1,96 @@ +package website + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "strconv" +) + +type Website struct { + Name string `json:"name"` + ApplicationPool string `json:"applicationPool"` + PhysicalPath string `json:"physicalPath"` + State string `json:"state"` +} + +func Run(commands string) (*string, error) { + + var stderr bytes.Buffer + var stdout bytes.Buffer + cmd := exec.Command("powershell.exe", "-ExecutionPolicy", "Bypass", "-NoLogo", "-NonInteractive", "-NoProfile", "-Command", commands) + cmd.Stderr = &stderr + cmd.Stdout = &stdout + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("error starting: %+v", err) + } + if err := cmd.Wait(); err != nil { + return nil, fmt.Errorf("error waiting: %+v", err) + } + stdOutStr := stdout.String() + + return &stdOutStr, nil +} + +func GetWorkerProcessId(appPool string) (int, error) { + + commands := fmt.Sprintf(` +Import-Module WebAdministration +$pids = dir "IIS:\AppPools\%s\WorkerProcesses" | Select-Object -expand processId | ConvertTo-Json + +if ($pids.Count -eq 0) { + Write-Host "[]" +} else { + if ($pids.Count -gt 1) { + $v = "[""{0}""]" -f $pids[0].ToString() + Write-Host $v + } else { + $v = $pids | ConvertTo-Json + Write-Host $v + } +} + `, appPool) + + stdout, err := Run(commands) + if err != nil { + return 0, fmt.Errorf("error retrieving Worker Process ID's for App Pool %q: %+v", appPool, err) + } + var stringId string + err = json.Unmarshal([]byte(*stdout), &stringId) + if err != nil { + return 0, fmt.Errorf("error parsing %q as a worker process id of type int: %+v", stdout, err) + } + id, err := strconv.Atoi(stringId) + return id, nil +} + +func GetApplicationPool(host string) (string, error) { + commands := fmt.Sprintf(` +Import-Module WebAdministration +Get-Website -Name %q | ConvertTo-Json -Compress + `, host) + + stdout, err := Run(commands) + if err != nil { + return "", fmt.Errorf("error retrieving website: %+v", err) + } + + var site Website + if out := stdout; out != nil && *out != "" { + v := *out + err := json.Unmarshal([]byte(v), &site) + if err != nil { + return "", fmt.Errorf("error unmarshalling website %q: %+v", host, err) + } + } + + if site.Name == "" { + return "", fmt.Errorf("website %q was not found", host) + } + if site.State == "Stopped" { + return "", fmt.Errorf("website %q has stopped", host) + } + + return site.ApplicationPool, nil +} diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index a1f1438669b5..a42c24ce555d 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" + "github.com/pkg/errors" ) // Config for the windows perfmon metricset. @@ -62,6 +63,37 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // Fetch fetches events and reports them upstream -func (m *MetricSet) Fetch(r mb.ReporterV2) error { +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before + if len(m.reader.query.Counters) == 0 { + return errors.New("no counters to read") + } + var config Config + if err := m.Module().UnpackConfig(&config); err != nil { + return nil + } + + // refresh performance counter list + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + // A flag is set if the second call has been executed else refresh will fail (reader.executed) + if m.reader.hasRun { + err := m.reader.InitCounters(config.Hosts) + if err != nil { + return errors.Wrap(err, "failed retrieving counters") + } + } + events, err := m.reader.Fetch() + if err != nil { + return errors.Wrap(err, "failed reading counters") + } + + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } + } + return nil } diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml.disabled similarity index 74% rename from metricbeat/modules.d/iis.yml rename to metricbeat/modules.d/iis.yml.disabled index 5593126a2501..dd8da07e529c 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml.disabled @@ -4,6 +4,8 @@ - module: iis metricsets: - webserver + - website + hosts: ["local.test"] enabled: true - period: 30s + period: 10s diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 0b34e0e04554..05064de0586a 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -4,9 +4,9 @@ - module: system period: 10s metricsets: - #- cpu - #- load - #- memory + - cpu + - load + - memory - network - process - process_summary From 411d9d901b5bfacfc1cceb362c87eac2c44d6cee Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 5 Feb 2020 13:17:16 +0100 Subject: [PATCH 18/36] adding application pool metricset --- metricbeat/docs/fields.asciidoc | 17 ++ metricbeat/docs/modules/iis.asciidoc | 7 +- .../modules/iis/application_pool.asciidoc | 23 ++ metricbeat/docs/modules_list.asciidoc | 3 +- .../helper/windows/pdh/defs_pdh_windows.go | 17 ++ .../windows/pdh/defs_pdh_windows_386.go | 17 ++ .../windows/pdh/defs_pdh_windows_amd64.go | 17 ++ metricbeat/helper/windows/pdh/zpdh_windows.go | 17 ++ metricbeat/include/list_common.go | 1 + metricbeat/metricbeat.reference.yml | 3 +- metricbeat/module/iis/_meta/config.yml | 3 +- .../iis/application_pool/_meta/data.json | 19 ++ .../iis/application_pool/_meta/docs.asciidoc | 1 + .../iis/application_pool/_meta/fields.yml | 10 + .../iis/application_pool/application_pool.go | 148 ++++++++++++ .../application_pool/application_pool_test.go | 18 ++ metricbeat/module/iis/application_pool/doc.go | 18 ++ metricbeat/module/iis/counters.go | 48 ++++ metricbeat/module/iis/doc.go | 1 + metricbeat/module/iis/fields.go | 2 +- metricbeat/module/iis/{website => }/reader.go | 148 +++++------- metricbeat/module/iis/webserver/manifest.yml | 222 +++++++++--------- metricbeat/module/iis/website/counters.go | 62 ----- metricbeat/module/iis/website/doc.go | 18 ++ metricbeat/module/iis/website/runner.go | 96 -------- metricbeat/module/iis/website/website.go | 106 ++++++--- metricbeat/modules.d/iis.yml.disabled | 3 +- metricbeat/modules.d/system.yml | 20 ++ 28 files changed, 667 insertions(+), 398 deletions(-) create mode 100644 metricbeat/docs/modules/iis/application_pool.asciidoc create mode 100644 metricbeat/module/iis/application_pool/_meta/data.json create mode 100644 metricbeat/module/iis/application_pool/_meta/docs.asciidoc create mode 100644 metricbeat/module/iis/application_pool/_meta/fields.yml create mode 100644 metricbeat/module/iis/application_pool/application_pool.go create mode 100644 metricbeat/module/iis/application_pool/application_pool_test.go create mode 100644 metricbeat/module/iis/application_pool/doc.go create mode 100644 metricbeat/module/iis/counters.go rename metricbeat/module/iis/{website => }/reader.go (52%) delete mode 100644 metricbeat/module/iis/website/counters.go create mode 100644 metricbeat/module/iis/website/doc.go delete mode 100644 metricbeat/module/iis/website/runner.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 810cb9a7fccd..d0a7b8907a4f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -15876,6 +15876,23 @@ iis module +[float] +=== application_pool + +application_pool + + + +*`iis.application_pool.example`*:: ++ +-- +Example field + + +type: keyword + +-- + [float] === webserver diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 8038fe6b8a77..244bf8946796 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -24,10 +24,9 @@ metricbeat.modules: metricsets: - webserver - website - hosts: ["local.test"] + - application_pool enabled: true period: 10s - ---- [float] @@ -35,10 +34,14 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> * <> +include::iis/application_pool.asciidoc[] + include::iis/webserver.asciidoc[] include::iis/website.asciidoc[] diff --git a/metricbeat/docs/modules/iis/application_pool.asciidoc b/metricbeat/docs/modules/iis/application_pool.asciidoc new file mode 100644 index 000000000000..3b1d78ab13f8 --- /dev/null +++ b/metricbeat/docs/modules/iis/application_pool.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-iis-application_pool]] +=== iis application_pool metricset + +beta[] + +include::../../../module/iis/application_pool/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/application_pool/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 6c48caef70cb..4a4010cb75c3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -104,7 +104,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.2+| .2+| |<> beta[] +.3+| .3+| |<> beta[] +|<> beta[] |<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] diff --git a/metricbeat/helper/windows/pdh/defs_pdh_windows.go b/metricbeat/helper/windows/pdh/defs_pdh_windows.go index 0df9a4736691..bcc62c4ffd1d 100644 --- a/metricbeat/helper/windows/pdh/defs_pdh_windows.go +++ b/metricbeat/helper/windows/pdh/defs_pdh_windows.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // go run mkpdh_defs.go // MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT diff --git a/metricbeat/helper/windows/pdh/defs_pdh_windows_386.go b/metricbeat/helper/windows/pdh/defs_pdh_windows_386.go index 3519e8f4e8f5..e794050dcf55 100644 --- a/metricbeat/helper/windows/pdh/defs_pdh_windows_386.go +++ b/metricbeat/helper/windows/pdh/defs_pdh_windows_386.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo.exe -godefs defs_pdh_windows.go diff --git a/metricbeat/helper/windows/pdh/defs_pdh_windows_amd64.go b/metricbeat/helper/windows/pdh/defs_pdh_windows_amd64.go index 3519e8f4e8f5..e794050dcf55 100644 --- a/metricbeat/helper/windows/pdh/defs_pdh_windows_amd64.go +++ b/metricbeat/helper/windows/pdh/defs_pdh_windows_amd64.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo.exe -godefs defs_pdh_windows.go diff --git a/metricbeat/helper/windows/pdh/zpdh_windows.go b/metricbeat/helper/windows/pdh/zpdh_windows.go index f730a016f372..15473a7f5a23 100644 --- a/metricbeat/helper/windows/pdh/zpdh_windows.go +++ b/metricbeat/helper/windows/pdh/zpdh_windows.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // Code generated by 'go generate'; DO NOT EDIT. package pdh diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 2df120dd089f..abaccc0e4bb4 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -77,6 +77,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" _ "github.com/elastic/beats/metricbeat/module/iis" + _ "github.com/elastic/beats/metricbeat/module/iis/application_pool" _ "github.com/elastic/beats/metricbeat/module/iis/website" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 5b3cc4e61f6f..a0d718c5df46 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -347,11 +347,10 @@ metricbeat.modules: metricsets: - webserver - website - hosts: ["local.test"] + - application_pool enabled: true period: 10s - #------------------------------- Jolokia Module ------------------------------- - module: jolokia #metricsets: ["jmx"] diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml index 3ff1e68e9d3c..7fff9dc3388a 100644 --- a/metricbeat/module/iis/_meta/config.yml +++ b/metricbeat/module/iis/_meta/config.yml @@ -2,7 +2,6 @@ metricsets: - webserver - website - hosts: ["local.test"] + - application_pool enabled: true period: 10s - diff --git a/metricbeat/module/iis/application_pool/_meta/data.json b/metricbeat/module/iis/application_pool/_meta/data.json new file mode 100644 index 000000000000..befe502b30db --- /dev/null +++ b/metricbeat/module/iis/application_pool/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"iis", + "name":"application_pool", + "rtt":44269 + }, + "iis":{ + "application_pool":{ + "example": "application_pool" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/iis/application_pool/_meta/docs.asciidoc b/metricbeat/module/iis/application_pool/_meta/docs.asciidoc new file mode 100644 index 000000000000..e8cd5b6d8413 --- /dev/null +++ b/metricbeat/module/iis/application_pool/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the application_pool metricset of the module iis. diff --git a/metricbeat/module/iis/application_pool/_meta/fields.yml b/metricbeat/module/iis/application_pool/_meta/fields.yml new file mode 100644 index 000000000000..3eec06cddaef --- /dev/null +++ b/metricbeat/module/iis/application_pool/_meta/fields.yml @@ -0,0 +1,10 @@ +- name: application_pool + type: group + release: beta + description: > + application_pool + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go new file mode 100644 index 000000000000..1f0cf3fc8067 --- /dev/null +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -0,0 +1,148 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package application_pool + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/iis" + "github.com/elastic/go-sysinfo" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("iis", "application_pool", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + log *logp.Logger + reader *iis.Reader +} + +// Config for the iis website metricset. +type Config struct { + Names []string `config:"name"` +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The iis application_pool metricset is beta.") + var config Config + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + reader, err := iis.NewReader() + if err != nil { + return nil, err + } + instances, err := getInstances(config.Names) + if err != nil { + return nil, err + } + reader.Instances = instances + if err := reader.InitCounters(nil, iis.AppPoolCounters); err != nil { + return nil, err + } + return &MetricSet{ + BaseMetricSet: base, + log: logp.NewLogger("application pool"), + reader: reader, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + var config Config + if err := m.Module().UnpackConfig(&config); err != nil { + return nil + } + instances, err := getInstances(config.Names) + if err != nil { + return err + } + m.reader.Instances = instances + events, err := m.reader.Fetch(nil, iis.AppPoolCounters) + if err != nil { + return errors.Wrap(err, "failed reading counters") + } + + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } + } + + return nil +} + +// getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users +func getInstances(names []string) ([]iis.Instance, error) { + processes, err := sysinfo.Processes() + if err != nil { + return nil, err + } + var w3processes []iis.Instance + for _, p := range processes { + info, err := p.Info() + if err != nil { + continue + } + if info.Name == "w3wp.exe" { + var app string + if len(info.Args) > 0 { + for i, ar := range info.Args { + if ar == "-ap" { + app = info.Args[i+1] + continue + } + } + } + w3processes = append(w3processes, iis.Instance{ProcessId: info.PID, Name: app}) + } + } + if len(names) == 0 { + return w3processes, nil + } + var filtered []iis.Instance + for _, n := range names { + for _, w3 := range w3processes { + if n == w3.Name { + filtered = append(filtered, w3) + } + } + } + return filtered, nil +} diff --git a/metricbeat/module/iis/application_pool/application_pool_test.go b/metricbeat/module/iis/application_pool/application_pool_test.go new file mode 100644 index 000000000000..3bc21201b3e7 --- /dev/null +++ b/metricbeat/module/iis/application_pool/application_pool_test.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package application_pool diff --git a/metricbeat/module/iis/application_pool/doc.go b/metricbeat/module/iis/application_pool/doc.go new file mode 100644 index 000000000000..3bc21201b3e7 --- /dev/null +++ b/metricbeat/module/iis/application_pool/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package application_pool diff --git a/metricbeat/module/iis/counters.go b/metricbeat/module/iis/counters.go new file mode 100644 index 000000000000..91480852ec6b --- /dev/null +++ b/metricbeat/module/iis/counters.go @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iis + +var ( + WebsiteCounters = map[string]string{ + "network.total_bytes_received": "\\Web Service(*)\\Total Bytes Received", + "network.total_bytes_sent": "\\Web Service(*)\\Total Bytes Sent", + "network.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", + "network.bytes_received_per_sec": "\\Web Service(*)\\Bytes Received/sec", + "network.current_connections": "\\Web Service(*)\\Current Connections", + "network.maximum_connections": "\\Web Service(*)\\Maximum Connections", + "network.total_connection_attempts": "\\Web Service(*)\\Total Connection Attempts (all instances)", + "network.total_get_requests": "\\Web Service(*)\\Total Get Requests", + "network.get_requests_per_sec": "\\Web Service(*)\\Get Requests/sec", + "network.total_post_requests": "\\Web Service(*)\\Total Post Requests", + "network.post_requests_per_sec": "\\Web Service(*)\\Post Requests/sec", + "network.total_delete_requests": "\\Web Service(*)\\Total Delete Requests", + "network.delete_requests_per_sec": "\\Web Service(*)\\Delete Requests/sec", + "network.service_uptime": "\\Web Service(*)\\Service Uptime", + } + AppPoolCounters = map[string]string{ + "process.cpu_usage_perc": "\\Process(*)\\% Processor Time", + "process.handle_count": "\\Process(*)\\Handle Count", + "process.thread_count": "\\Process(*)\\Thread Count", + "process.working_set": "\\Process(*)\\Working Set", + "process.private_byte": "\\Process(*)\\Private Bytes", + "process.virtual_bytes": "\\Process(*)\\Virtual Bytes", + "process.page_faults_per_sec": "\\Process(*)\\Page Faults/sec", + "process.io_read_operations_per_sec": "\\Process(*)\\IO Read Operations/sec", + "process.io_write_operations_per_sec": "\\Process(*)\\IO Write Operations/sec", + } +) diff --git a/metricbeat/module/iis/doc.go b/metricbeat/module/iis/doc.go index 3b68541b51c1..3cd0f406ee2a 100644 --- a/metricbeat/module/iis/doc.go +++ b/metricbeat/module/iis/doc.go @@ -16,4 +16,5 @@ // under the License. // Package iis is a Metricbeat module that contains MetricSets. + package iis diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index 0f5cc7f6221e..877c832d515d 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -32,5 +32,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/iis. func AssetIis() string { - return "eJykz0kOwjAMheG9T/HUfS/gBXdp6QNZpINiF9TbozKJSGGFl4n1/XKLCzeFmQsQFomKxswbAQb6MdsSNk+KgwDY9zDOw5ooQGZi51T0jE4EOBnT4PrYbDF1I9/yPrEtVJzzvC6vl0qgRL6hG3tnvjJ/fmrkT/Y5NaQ8o5a14L/Rkqgn5R4AAP//OjVpIQ==" + return "eJykkD1uxCAUhHtOMdp+L0CRLueI2GUSPRkbBDiObx9h8oMd5BQ7JTx932iuGLhqiCQFZMmOGheRdFGAZbpHCVn8pPGkAJQ7jN7OjgqIdDSJGjdmoxTwKnQ26e3yismM/CaX5DVQ4y36OXy9dAR7SAsyITi5m3L8Erx3Pwc9csm+3u9711pz4jjWaqvxw4xhm6RNbTVwXXy0h7+TDiXPFVilf5ZYeEuM74z/TnCi6UH6i7VayXxUukf0leozAAD//xsprTU=" } diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/reader.go similarity index 52% rename from metricbeat/module/iis/website/reader.go rename to metricbeat/module/iis/reader.go index d4b74df38bb6..1eac69fca202 100644 --- a/metricbeat/module/iis/website/reader.go +++ b/metricbeat/module/iis/reader.go @@ -17,84 +17,69 @@ // +build windows -package website +package iis import ( + "strings" + + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper/windows/pdh" "github.com/elastic/beats/metricbeat/mb" - "github.com/pkg/errors" - "strings" ) // Reader will contain the config options type Reader struct { - query pdh.Query // PDH Query - hosts []Host // Mapping of counter path to key used for the label (e.g. processor.name) - log *logp.Logger // logger - hasRun bool // will check if the reader has run a first time + Query pdh.Query // PDH Query + Instances []Instance // Mapping of counter path to key used for the label (e.g. processor.name) + log *logp.Logger // logger + hasRun bool // will check if the reader has run a first time } -type Host struct { - name string - processId int +type Instance struct { + Name string + ProcessId int counters map[string]string } // NewReader creates a new instance of Reader. -func NewReader(config Config) (*Reader, error) { +func NewReader() (*Reader, error) { var query pdh.Query if err := query.Open(); err != nil { return nil, err } reader := &Reader{ - query: query, + Query: query, log: logp.NewLogger("website"), } - if err := reader.InitCounters(config.Hosts); err != nil { - return nil, err - } + return reader, nil } -func (this *Reader) InitCounters(hosts []string) error { - counters, instances, err := this.query.GetCountersAndInstances("Web Service") - _ = counters - if err != nil { - this.query.Close() - return err - } - this.hosts = filterOnInstances(hosts, instances) +func (re *Reader) InitCounters(nameCounters map[string]string, processIdCounters map[string]string) error { var newQueries []string - for i, instance := range this.hosts { - this.hosts[i].counters = make(map[string]string) - for key, value := range webserverCounters { - value = strings.Replace(value, "*", instance.name, 1) - if err := this.query.AddCounter(value, "", "float", true); err != nil { + for i, instance := range re.Instances { + re.Instances[i].counters = make(map[string]string) + for key, value := range nameCounters { + value = strings.Replace(value, "*", instance.Name, 1) + if err := re.Query.AddCounter(value, "", "float", true); err != nil { return errors.Wrapf(err, `failed to add counter (query="%v")`, value) } newQueries = append(newQueries, value) - this.hosts[i].counters[value] = key + re.Instances[i].counters[value] = key } - var processId int - processId, err = GetProcessId(instance.name) - this.hosts[i].processId = processId - if err != nil { - this.log.Errorf("Cannot find attached worker process %s", err) - continue - } - for key, value := range processCounters { - value = strings.Replace(value, "*", string(processId), 1) - if err := this.query.AddCounter(value, "", "float", true); err != nil { + for key, value := range processIdCounters { + value = strings.Replace(value, "*", string(instance.ProcessId), 1) + if err := re.Query.AddCounter(value, "", "float", true); err != nil { return errors.Wrapf(err, `failed to add counter (query="%v")`, value) } newQueries = append(newQueries, value) - this.hosts[i].counters[value] = key + re.Instances[i].counters[value] = key } } - - err = this.query.RemoveUnusedCounters(newQueries) + err := re.Query.RemoveUnusedCounters(newQueries) if err != nil { return errors.Wrap(err, "failed removing unused counter values") } @@ -102,43 +87,59 @@ func (this *Reader) InitCounters(hosts []string) error { } // Read executes a query and returns those values in an event. -func (this *Reader) Fetch() ([]mb.Event, error) { +func (re *Reader) Fetch(nameCounters map[string]string, processIdCounters map[string]string) ([]mb.Event, error) { + // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before + if len(re.Query.Counters) == 0 { + return nil, errors.New("no counters to read") + } + + // refresh performance counter list + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + // A flag is set if the second call has been executed else refresh will fail (reader.executed) + if re.hasRun { + err := re.InitCounters(nameCounters, processIdCounters) + if err != nil { + return nil, errors.Wrap(err, "failed retrieving counters") + } + } + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if err := this.query.CollectData(); err != nil { + if err := re.Query.CollectData(); err != nil { return nil, errors.Wrap(err, "failed querying counter values") } // Get the values. - values, err := this.query.GetFormattedCounterValues() + values, err := re.Query.GetFormattedCounterValues() if err != nil { return nil, errors.Wrap(err, "failed formatting counter values") } events := make(map[string]mb.Event) - for _, host := range this.hosts { - events[host.name] = mb.Event{ + for _, host := range re.Instances { + events[host.Name] = mb.Event{ MetricSetFields: common.MapStr{ - "name": host.name, - "worker_process_id": host.processId, + "name": host.Name, + "worker_process_id": host.ProcessId, }, } for counterPath, values := range values { for _, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !this.hasRun { - this.log.Debugw("Ignoring the first measurement because the data isn't ready", + if val.Err != nil && !re.hasRun { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", "error", val.Err, logp.Namespace("website"), "query", counterPath) continue } - if val.Instance == host.name || val.Instance == string(host.processId) { - events[host.name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) + if val.Instance == host.Name || val.Instance == string(host.ProcessId) { + events[host.Name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) } } } } - this.hasRun = true + re.hasRun = true results := make([]mb.Event, 0, len(events)) for _, val := range events { results = append(results, val) @@ -147,41 +148,6 @@ func (this *Reader) Fetch() ([]mb.Event, error) { } // Close will close the PDH query for now. -func (this *Reader) Close() error { - return this.query.Close() -} - -func filterOnInstances(hosts []string, instances []string) []Host { - filtered := make([]Host, 0) - // remove _Total and empty instances - for _, instance := range instances { - if instance == "_Total" || instance == "" { - continue - } - if containsHost(instance, hosts) { - filtered = append(filtered, Host{name: instance}) - } - } - return filtered -} - -func containsHost(item string, array []string) bool { - // if no hosts specified all instances are selected - if len(array) == 0 { - return true - } - for _, i := range array { - if i == item { - return true - } - } - return false -} - -func GetProcessId(host string) (int, error) { - appPool, err := GetApplicationPool(host) - if err != nil { - return 0, err - } - return GetWorkerProcessId(appPool) +func (re *Reader) Close() error { + return re.Query.Close() } diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 3ed0eb25f5bf..23d16696891a 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -8,116 +8,116 @@ input: perfmon.group_all_counter: true perfmon.counters: #network - - instance_label: '' - measurement_label: network.total_bytes_received - query: '\Web Service(_Total)\Total Bytes Received' - - instance_label: '' - measurement_label: network.total_bytes_sent - query: '\Web Service(_Total)\Total Bytes Sent' - - instance_label: '' - measurement_label: network.bytes_sent_per_sec - query: '\Web Service(_Total)\Bytes Sent/sec' - - instance_label: '' - measurement_label: network.bytes_received_per_sec - query: '\Web Service(_Total)\Bytes Received/sec' - - instance_label: '' - measurement_label: network.current_connections - query: 'Web Service(_Total)\Current Connections' - - instance_label: '' - measurement_label: network.maximum_connections - query: '\Web Service(_Total)\Maximum Connections' - - instance_label: '' - measurement_label: network.total_connection_attempts - query: '\Web Service(_Total)\Total Connection Attempts (all instances)' - - instance_label: '' - measurement_label: network.total_get_requests - query: '\Web Service(_Total)\Total Get Requests' - - instance_label: '' - measurement_label: network.get_requests_per_sec - query: '\Web Service(_Total)\Get Requests/sec' - - instance_label: '' - measurement_label: network.total_post_requests - query: '\Web Service(_Total)\Total Post Requests' - - instance_label: '' - measurement_label: network.post_requests_per_sec - query: '\Web Service(_Total)\Post Requests/sec' - - instance_label: '' - measurement_label: network.total_delete_requests - query: '\Web Service(_Total)\Total Delete Requests' - - instance_label: '' - measurement_label: network.delete_requests_per_sec - query: '\Web Service(_Total)\Delete Requests/sec' - - instance_label: '' - measurement_label: network.service_uptime - query: '\Web Service(_Total)\Service Uptime' + - instance_label: '' + measurement_label: network.total_bytes_received + query: '\Web Service(_Total)\Total Bytes Received' + - instance_label: '' + measurement_label: network.total_bytes_sent + query: '\Web Service(_Total)\Total Bytes Sent' + - instance_label: '' + measurement_label: network.bytes_sent_per_sec + query: '\Web Service(_Total)\Bytes Sent/sec' + - instance_label: '' + measurement_label: network.bytes_received_per_sec + query: '\Web Service(_Total)\Bytes Received/sec' + - instance_label: '' + measurement_label: network.current_connections + query: 'Web Service(_Total)\Current Connections' + - instance_label: '' + measurement_label: network.maximum_connections + query: '\Web Service(_Total)\Maximum Connections' + - instance_label: '' + measurement_label: network.total_connection_attempts + query: '\Web Service(_Total)\Total Connection Attempts (all instances)' + - instance_label: '' + measurement_label: network.total_get_requests + query: '\Web Service(_Total)\Total Get Requests' + - instance_label: '' + measurement_label: network.get_requests_per_sec + query: '\Web Service(_Total)\Get Requests/sec' + - instance_label: '' + measurement_label: network.total_post_requests + query: '\Web Service(_Total)\Total Post Requests' + - instance_label: '' + measurement_label: network.post_requests_per_sec + query: '\Web Service(_Total)\Post Requests/sec' + - instance_label: '' + measurement_label: network.total_delete_requests + query: '\Web Service(_Total)\Total Delete Requests' + - instance_label: '' + measurement_label: network.delete_requests_per_sec + query: '\Web Service(_Total)\Delete Requests/sec' + - instance_label: '' + measurement_label: network.service_uptime + query: '\Web Service(_Total)\Service Uptime' #cache - - instance_label: '' - measurement_label: cache.current_files_cached - query: '\Web Service Cache\Current Files Cached' - - instance_label: '' - measurement_label: cache.total_files_cached - query: '\Web Service Cache\Total Files Cached' - - instance_label: '' - measurement_label: cache.file_cache_hits - query: '\Web Service Cache\File Cache Hits' - - instance_label: '' - measurement_label: cache.file_cache_misses - query: '\Web Service Cache\File Cache Misses' - - instance_label: '' - measurement_label: cache.current_file_cache_memory_usage - query: '\Web Service Cache\Current File Cache Memory Usage' - - instance_label: '' - measurement_label: cache.maximum_file_cache_memory_usage - query: '\Web Service Cache\Maximum File Cache Memory Usage' - - instance_label: '' - measurement_label: cache.current_uris_cached - query: '\Web Service Cache\Current URIs Cached' - - instance_label: '' - measurement_label: cache.total_uris_cached - query: '\Web Service Cache\Total URIs Cached' - - instance_label: '' - measurement_label: cache.uri_cache_hits - query: '\Web Service Cache\URI Cache Hits' - - instance_label: '' - measurement_label: cache.uri_cache_misses - query: '\Web Service Cache\URI Cache Misses' - - instance_label: '' - measurement_label: cache.output_cache_current_memory_usage - query: '\Web Service Cache\Output Cache Current Memory Usage' - - instance_label: '' - measurement_label: cache.output_cache_current_items - query: '\Web Service Cache\Output Cache Current Items' - - instance_label: '' - measurement_label: cache.output_cache_total_hits - query: '\Web Service Cache\Output Cache Total Hits' - - instance_label: '' - measurement_label: cache.output_cache_total_misses - query: '\Web Service Cache\Output Cache Total Misses' + - instance_label: '' + measurement_label: cache.current_files_cached + query: '\Web Service Cache\Current Files Cached' + - instance_label: '' + measurement_label: cache.total_files_cached + query: '\Web Service Cache\Total Files Cached' + - instance_label: '' + measurement_label: cache.file_cache_hits + query: '\Web Service Cache\File Cache Hits' + - instance_label: '' + measurement_label: cache.file_cache_misses + query: '\Web Service Cache\File Cache Misses' + - instance_label: '' + measurement_label: cache.current_file_cache_memory_usage + query: '\Web Service Cache\Current File Cache Memory Usage' + - instance_label: '' + measurement_label: cache.maximum_file_cache_memory_usage + query: '\Web Service Cache\Maximum File Cache Memory Usage' + - instance_label: '' + measurement_label: cache.current_uris_cached + query: '\Web Service Cache\Current URIs Cached' + - instance_label: '' + measurement_label: cache.total_uris_cached + query: '\Web Service Cache\Total URIs Cached' + - instance_label: '' + measurement_label: cache.uri_cache_hits + query: '\Web Service Cache\URI Cache Hits' + - instance_label: '' + measurement_label: cache.uri_cache_misses + query: '\Web Service Cache\URI Cache Misses' + - instance_label: '' + measurement_label: cache.output_cache_current_memory_usage + query: '\Web Service Cache\Output Cache Current Memory Usage' + - instance_label: '' + measurement_label: cache.output_cache_current_items + query: '\Web Service Cache\Output Cache Current Items' + - instance_label: '' + measurement_label: cache.output_cache_total_hits + query: '\Web Service Cache\Output Cache Total Hits' + - instance_label: '' + measurement_label: cache.output_cache_total_misses + query: '\Web Service Cache\Output Cache Total Misses' #process - - instance_label: '' - measurement_label: process.cpu_usage_perc - query: '\Process(w3wp*)\% Processor Time' - - instance_label: '' - measurement_label: process.handle_count - query: '\Process(w3wp*)\Handle Count' - - instance_label: '' - measurement_label: process.thread_count - query: '\Process(w3wp*)\Thread Count' - - instance_label: '' - measurement_label: process.working_set - query: '\Process(w3wp*)\Working Set' - - instance_label: '' - measurement_label: process.private_byte - query: '\Process(w3wp*)\Private Bytes' - - instance_label: '' - measurement_label: process.virtual_bytes - query: '\Process(w3wp*)\Virtual Bytes' - - instance_label: '' - measurement_label: process.page_faults_per_Sec - query: '\Process(w3wp*)\Page Faults/sec' - - instance_label: '' - measurement_label: process.io_read_operations_per_sec - query: '\Process(w3wp*)\IO Read Operations/sec' - - instance_label: '' - measurement_label: process.io_write_operations_per_sec - query: '\Process(w3wp*)\IO Write Operations/sec' + - instance_label: '' + measurement_label: process.cpu_usage_perc + query: '\Process(w3wp*)\% Processor Time' + - instance_label: '' + measurement_label: process.handle_count + query: '\Process(w3wp*)\Handle Count' + - instance_label: '' + measurement_label: process.thread_count + query: '\Process(w3wp*)\Thread Count' + - instance_label: '' + measurement_label: process.working_set + query: '\Process(w3wp*)\Working Set' + - instance_label: '' + measurement_label: process.private_byte + query: '\Process(w3wp*)\Private Bytes' + - instance_label: '' + measurement_label: process.virtual_bytes + query: '\Process(w3wp*)\Virtual Bytes' + - instance_label: '' + measurement_label: process.page_faults_per_Sec + query: '\Process(w3wp*)\Page Faults/sec' + - instance_label: '' + measurement_label: process.io_read_operations_per_sec + query: '\Process(w3wp*)\IO Read Operations/sec' + - instance_label: '' + measurement_label: process.io_write_operations_per_sec + query: '\Process(w3wp*)\IO Write Operations/sec' diff --git a/metricbeat/module/iis/website/counters.go b/metricbeat/module/iis/website/counters.go deleted file mode 100644 index 708398eb6bc8..000000000000 --- a/metricbeat/module/iis/website/counters.go +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package website - -var ( - webserverCounters = map[string]string{ - "network.total_bytes_received": "\\Web Service(*)\\Total Bytes Received", - "network.total_bytes_sent": "\\Web Service(*)\\Total Bytes Sent", - "network.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", - "network.bytes_received_per_sec": "\\Web Service(*)\\Bytes Received/sec", - "network.current_connections": "\\Web Service(*)\\Current Connections", - "network.maximum_connections": "\\Web Service(*)\\Maximum Connections", - "network.total_connection_attempts": "\\Web Service(*)\\Total Connection Attempts (all instances)", - "network.total_get_requests": "\\Web Service(*)\\Total Get Requests", - "network.get_requests_per_sec": "\\Web Service(*)\\Get Requests/sec", - "network.total_post_requests": "\\Web Service(*)\\Total Post Requests", - "network.post_requests_per_sec": "\\Web Service(*)\\Post Requests/sec", - "network.total_delete_requests": "\\Web Service(*)\\Total Delete Requests", - "network.delete_requests_per_sec": "\\Web Service(*)\\Delete Requests/sec", - "network.service_uptime": "\\Web Service(*)\\Service Uptime", - "cache.current_files_cached": "\\Web Service Cache\\Current Files Cached", - "cache.total_files_cached": "\\Web Service Cache\\Total Files Cached", - "cache.file_cache_hits": "\\Web Service Cache\\File Cache Hits", - "cache.file_cache_misses": "\\Web Service Cache\\File Cache Misses", - "cache.current_file_cache_memory_usage": "\\Web Service Cache\\Current File Cache Memory Usage", - "cache.maximum_file_cache_memory_usage": "\\Web Service Cache\\Maximum File Cache Memory Usage", - "cache.current_uris_cached": "\\Web Service Cache\\Current URIs Cached", - "cache.total_uris_cached": "\\Web Service Cache\\Total URIs Cached", - "cache.uri_cache_hits": "\\Web Service Cache\\URI Cache Hits", - "cache.uri_cache_misses": "\\Web Service Cache\\URI Cache Misses", - "cache.output_cache_current_memory_usage": "\\Web Service Cache\\Output Cache Current Memory Usage", - "cache.output_cache_current_items": "\\Web Service Cache\\Output Cache Current Items", - "cache.output_cache_total_hits": "\\Web Service Cache\\Output Cache Total Hits", - "cache.output_cache_total_misses": "\\Web Service Cache\\Output Cache Total Misses", - } - processCounters = map[string]string{ - "process.cpu_usage_perc": "\\Process(*)\\% Processor Time", - "process.handle_count": "\\Process(*)\\Handle Count", - "process.thread_count": "\\Process(*)\\Thread Count", - "process.working_set": "\\Process(*)\\Working Set", - "process.private_byte": "\\Process(*)\\Private Bytes", - "process.virtual_bytes": "\\Process(*)\\Virtual Bytes", - "process.page_faults_per_sec": "\\Process(*)\\Page Faults/sec", - "process.io_read_operations_per_sec": "\\Process(*)\\IO Read Operations/sec", - "process.io_write_operations_per_sec": "\\Process(*)\\IO Write Operations/sec", - } -) diff --git a/metricbeat/module/iis/website/doc.go b/metricbeat/module/iis/website/doc.go new file mode 100644 index 000000000000..81e02fe557b4 --- /dev/null +++ b/metricbeat/module/iis/website/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website diff --git a/metricbeat/module/iis/website/runner.go b/metricbeat/module/iis/website/runner.go deleted file mode 100644 index 4fa4110ff309..000000000000 --- a/metricbeat/module/iis/website/runner.go +++ /dev/null @@ -1,96 +0,0 @@ -package website - -import ( - "bytes" - "encoding/json" - "fmt" - "os/exec" - "strconv" -) - -type Website struct { - Name string `json:"name"` - ApplicationPool string `json:"applicationPool"` - PhysicalPath string `json:"physicalPath"` - State string `json:"state"` -} - -func Run(commands string) (*string, error) { - - var stderr bytes.Buffer - var stdout bytes.Buffer - cmd := exec.Command("powershell.exe", "-ExecutionPolicy", "Bypass", "-NoLogo", "-NonInteractive", "-NoProfile", "-Command", commands) - cmd.Stderr = &stderr - cmd.Stdout = &stdout - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("error starting: %+v", err) - } - if err := cmd.Wait(); err != nil { - return nil, fmt.Errorf("error waiting: %+v", err) - } - stdOutStr := stdout.String() - - return &stdOutStr, nil -} - -func GetWorkerProcessId(appPool string) (int, error) { - - commands := fmt.Sprintf(` -Import-Module WebAdministration -$pids = dir "IIS:\AppPools\%s\WorkerProcesses" | Select-Object -expand processId | ConvertTo-Json - -if ($pids.Count -eq 0) { - Write-Host "[]" -} else { - if ($pids.Count -gt 1) { - $v = "[""{0}""]" -f $pids[0].ToString() - Write-Host $v - } else { - $v = $pids | ConvertTo-Json - Write-Host $v - } -} - `, appPool) - - stdout, err := Run(commands) - if err != nil { - return 0, fmt.Errorf("error retrieving Worker Process ID's for App Pool %q: %+v", appPool, err) - } - var stringId string - err = json.Unmarshal([]byte(*stdout), &stringId) - if err != nil { - return 0, fmt.Errorf("error parsing %q as a worker process id of type int: %+v", stdout, err) - } - id, err := strconv.Atoi(stringId) - return id, nil -} - -func GetApplicationPool(host string) (string, error) { - commands := fmt.Sprintf(` -Import-Module WebAdministration -Get-Website -Name %q | ConvertTo-Json -Compress - `, host) - - stdout, err := Run(commands) - if err != nil { - return "", fmt.Errorf("error retrieving website: %+v", err) - } - - var site Website - if out := stdout; out != nil && *out != "" { - v := *out - err := json.Unmarshal([]byte(v), &site) - if err != nil { - return "", fmt.Errorf("error unmarshalling website %q: %+v", host, err) - } - } - - if site.Name == "" { - return "", fmt.Errorf("website %q was not found", host) - } - if site.State == "Stopped" { - return "", fmt.Errorf("website %q has stopped", host) - } - - return site.ApplicationPool, nil -} diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index a42c24ce555d..3f99903bfb8d 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -20,30 +20,39 @@ package website import ( + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" - "github.com/pkg/errors" + "github.com/elastic/beats/metricbeat/module/iis" ) -// Config for the windows perfmon metricset. -type Config struct { - Hosts []string `config:"hosts"` -} - -// init registers the partition MetricSet with the central registry. +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("iis", "website", New) + mb.Registry.AddMetricSet("iis", "website", New) } -// MetricSet type defines all fields of the partition MetricSet +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet log *logp.Logger - reader *Reader + reader *iis.Reader +} + +// Config for the iis website metricset. +type Config struct { + Sites []string `config:"site"` } -// New creates a new instance of the website MetricSet. +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Beta("The website metricset is beta") @@ -51,10 +60,20 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - reader, err := NewReader(config) + // init reader object + reader, err := iis.NewReader() if err != nil { return nil, err } + // get instances will retrieve the websites running in iis and also filter on the sites configured by users + instances, err := getInstances(reader, config.Sites) + if err != nil { + return nil, err + } + reader.Instances = instances + if err := reader.InitCounters(iis.WebsiteCounters, nil); err != nil { + return nil, err + } return &MetricSet{ BaseMetricSet: base, log: logp.NewLogger("website"), @@ -62,28 +81,20 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } -// Fetch fetches events and reports them upstream +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { - // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before - if len(m.reader.query.Counters) == 0 { - return errors.New("no counters to read") - } var config Config if err := m.Module().UnpackConfig(&config); err != nil { return nil } - - // refresh performance counter list - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - // A flag is set if the second call has been executed else refresh will fail (reader.executed) - if m.reader.hasRun { - err := m.reader.InitCounters(config.Hosts) - if err != nil { - return errors.Wrap(err, "failed retrieving counters") - } + instances, err := getInstances(m.reader, config.Sites) + if err != nil { + return err } - events, err := m.reader.Fetch() + m.reader.Instances = instances + events, err := m.reader.Fetch(iis.WebsiteCounters, nil) if err != nil { return errors.Wrap(err, "failed reading counters") } @@ -97,3 +108,42 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } + +// getInstances will retrieve the websites running in iis and also filter on the sites configured by users +func getInstances(reader *iis.Reader, sites []string) ([]iis.Instance, error) { + _, instances, err := reader.Query.GetCountersAndInstances("Web Service") + if err != nil { + reader.Query.Close() + return nil, err + } + return filterOnInstances(sites, instances), nil +} + +// filterOnInstances will filter on the sites configured by users +func filterOnInstances(hosts []string, instances []string) []iis.Instance { + filtered := make([]iis.Instance, 0) + // remove _Total and empty instances + for _, instance := range instances { + if instance == "_Total" || instance == "" { + continue + } + if containsHost(instance, hosts) { + filtered = append(filtered, iis.Instance{Name: instance}) + } + } + return filtered +} + +// containsHost checks if array contains site/hostname +func containsHost(item string, array []string) bool { + // if no hosts specified all instances are selected + if len(array) == 0 { + return true + } + for _, i := range array { + if i == item { + return true + } + } + return false +} diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml.disabled index dd8da07e529c..b8091f97cbd6 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml.disabled @@ -5,7 +5,6 @@ metricsets: - webserver - website - hosts: ["local.test"] + - application_pool enabled: true period: 10s - diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 05064de0586a..603dee8c0a45 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -19,3 +19,23 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + +- module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + +- module: system + period: 15m + metricsets: + - uptime + +#- module: system +# period: 5m +# metricsets: +# - raid +# raid.mount_point: '/' From 113eec556f0576ccc5c5a4a94dd62953e07a23b2 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 10 Feb 2020 11:31:10 +0100 Subject: [PATCH 19/36] wmi option --- .../helper/windows/pdh/pdh_query_windows.go | 6 +- .../iis/application_pool/application_pool.go | 40 +-- metricbeat/module/iis/counters.go | 25 +- metricbeat/module/iis/reader.go | 66 +++-- metricbeat/module/iis/website/_meta/data.json | 48 ++-- metricbeat/module/iis/website/reader.go | 234 ++++++++++++++++++ metricbeat/module/iis/website/website.go | 20 +- .../modules.d/{iis.yml.disabled => iis.yml} | 4 +- .../{system.yml => system.yml.disabled} | 0 9 files changed, 364 insertions(+), 79 deletions(-) create mode 100644 metricbeat/module/iis/website/reader.go rename metricbeat/modules.d/{iis.yml.disabled => iis.yml} (75%) rename metricbeat/modules.d/{system.yml => system.yml.disabled} (100%) diff --git a/metricbeat/helper/windows/pdh/pdh_query_windows.go b/metricbeat/helper/windows/pdh/pdh_query_windows.go index a2008ed8bfb6..898d5573a60a 100644 --- a/metricbeat/helper/windows/pdh/pdh_query_windows.go +++ b/metricbeat/helper/windows/pdh/pdh_query_windows.go @@ -80,7 +80,7 @@ func (q *Query) AddCounter(counterPath string, instance string, format string, w var instanceName string // Extract the instance name from the counterPath. if instance == "" || wildcard { - instanceName, err = matchInstanceName(counterPath) + instanceName, err = MatchInstanceName(counterPath) if err != nil { return err } @@ -221,8 +221,8 @@ func (q *Query) Close() error { return PdhCloseQuery(q.Handle) } -// matchInstanceName will check first for instance and then for any objects names. -func matchInstanceName(counterPath string) (string, error) { +// MatchInstanceName will check first for instance and then for any objects names. +func MatchInstanceName(counterPath string) (string, error) { matches := instanceNameRegexp.FindStringSubmatch(counterPath) if len(matches) != 2 { matches = objectNameRegexp.FindStringSubmatch(counterPath) diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index 1f0cf3fc8067..f237346e5d5b 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -20,13 +20,12 @@ package application_pool import ( - "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/iis" "github.com/elastic/go-sysinfo" + "github.com/pkg/errors" ) // init registers the MetricSet with the central registry as soon as the program @@ -43,8 +42,9 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - log *logp.Logger - reader *iis.Reader + log *logp.Logger + reader *iis.Reader + instanceReader *iis.Reader } // Config for the iis website metricset. @@ -60,23 +60,30 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - + // instantiate reader object reader, err := iis.NewReader() if err != nil { return nil, err } - instances, err := getInstances(config.Names) + // instantiate reader object that should retrieve the process ids and process names for the worker processes + instanceReader, err := iis.NewReader() + if err != nil { + return nil, err + } + + instances, err := getInstances(config.Names, instanceReader) if err != nil { return nil, err } reader.Instances = instances - if err := reader.InitCounters(nil, iis.AppPoolCounters); err != nil { + if err := reader.InitCounters(iis.AppPoolCounters); err != nil { return nil, err } return &MetricSet{ - BaseMetricSet: base, - log: logp.NewLogger("application pool"), - reader: reader, + BaseMetricSet: base, + log: logp.NewLogger("application pool"), + reader: reader, + instanceReader: instanceReader, }, nil } @@ -88,12 +95,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err := m.Module().UnpackConfig(&config); err != nil { return nil } - instances, err := getInstances(config.Names) + instances, err := getInstances(config.Names, m.instanceReader) if err != nil { return err } m.reader.Instances = instances - events, err := m.reader.Fetch(nil, iis.AppPoolCounters) + events, err := m.reader.Fetch(iis.AppPoolCounters) if err != nil { return errors.Wrap(err, "failed reading counters") } @@ -109,7 +116,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users -func getInstances(names []string) ([]iis.Instance, error) { +func getInstances(names []string, instanceReader *iis.Reader) ([]iis.Instance, error) { processes, err := sysinfo.Processes() if err != nil { return nil, err @@ -134,7 +141,7 @@ func getInstances(names []string) ([]iis.Instance, error) { } } if len(names) == 0 { - return w3processes, nil + return instanceReader.GetInstances(w3processes) } var filtered []iis.Instance for _, n := range names { @@ -144,5 +151,8 @@ func getInstances(names []string) ([]iis.Instance, error) { } } } - return filtered, nil + return instanceReader.GetInstances(filtered) } + + + diff --git a/metricbeat/module/iis/counters.go b/metricbeat/module/iis/counters.go index 91480852ec6b..5d2efe530410 100644 --- a/metricbeat/module/iis/counters.go +++ b/metricbeat/module/iis/counters.go @@ -35,14 +35,21 @@ var ( "network.service_uptime": "\\Web Service(*)\\Service Uptime", } AppPoolCounters = map[string]string{ - "process.cpu_usage_perc": "\\Process(*)\\% Processor Time", - "process.handle_count": "\\Process(*)\\Handle Count", - "process.thread_count": "\\Process(*)\\Thread Count", - "process.working_set": "\\Process(*)\\Working Set", - "process.private_byte": "\\Process(*)\\Private Bytes", - "process.virtual_bytes": "\\Process(*)\\Virtual Bytes", - "process.page_faults_per_sec": "\\Process(*)\\Page Faults/sec", - "process.io_read_operations_per_sec": "\\Process(*)\\IO Read Operations/sec", - "process.io_write_operations_per_sec": "\\Process(*)\\IO Write Operations/sec", + "worker_process_id": "\\Process(w3wp*)\\ID Process", + "process.cpu_usage_perc": "\\Process(w3wp*)\\% Processor Time", + "process.handle_count": "\\Process(w3wp*)\\Handle Count", + "process.thread_count": "\\Process(w3wp*)\\Thread Count", + "process.working_set": "\\Process(w3wp*)\\Working Set", + "process.private_byte": "\\Process(w3wp*)\\Private Bytes", + "process.virtual_bytes": "\\Process(w3wp*)\\Virtual Bytes", + "process.page_faults_per_sec": "\\Process(w3wp*)\\Page Faults/sec", + "process.io_read_operations_per_sec": "\\Process(w3wp*)\\IO Read Operations/sec", + "process.io_write_operations_per_sec": "\\Process(w3wp*)\\IO Write Operations/sec", + "net_clr.total_exceptions_thrown": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown", + "net_clr.exceptions_thrown_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown / sec", + "net_clr.filters_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Filters / sec", + "net_clr.finallys_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Finallys / sec", + "net_clr.throw_to_catch_depth_per_sec": "\\.NET CLR Exceptions(w3wp*)\\Throw To Catch Depth / sec", + //"asp_net_applications.request_bytes_total": "\\ASP.NET Applications(w3wp*)\\Request Bytes In Total", } ) diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/reader.go index 1eac69fca202..5848cff84b01 100644 --- a/metricbeat/module/iis/reader.go +++ b/metricbeat/module/iis/reader.go @@ -39,9 +39,10 @@ type Reader struct { } type Instance struct { - Name string - ProcessId int - counters map[string]string + Name string + ProcessId int + ProcessName string + counters map[string]string } // NewReader creates a new instance of Reader. @@ -58,11 +59,11 @@ func NewReader() (*Reader, error) { return reader, nil } -func (re *Reader) InitCounters(nameCounters map[string]string, processIdCounters map[string]string) error { +func (re *Reader) InitCounters(counters map[string]string) error { var newQueries []string for i, instance := range re.Instances { re.Instances[i].counters = make(map[string]string) - for key, value := range nameCounters { + for key, value := range counters { value = strings.Replace(value, "*", instance.Name, 1) if err := re.Query.AddCounter(value, "", "float", true); err != nil { return errors.Wrapf(err, `failed to add counter (query="%v")`, value) @@ -70,14 +71,6 @@ func (re *Reader) InitCounters(nameCounters map[string]string, processIdCounters newQueries = append(newQueries, value) re.Instances[i].counters[value] = key } - for key, value := range processIdCounters { - value = strings.Replace(value, "*", string(instance.ProcessId), 1) - if err := re.Query.AddCounter(value, "", "float", true); err != nil { - return errors.Wrapf(err, `failed to add counter (query="%v")`, value) - } - newQueries = append(newQueries, value) - re.Instances[i].counters[value] = key - } } err := re.Query.RemoveUnusedCounters(newQueries) if err != nil { @@ -87,7 +80,7 @@ func (re *Reader) InitCounters(nameCounters map[string]string, processIdCounters } // Read executes a query and returns those values in an event. -func (re *Reader) Fetch(nameCounters map[string]string, processIdCounters map[string]string) ([]mb.Event, error) { +func (re *Reader) Fetch(counters map[string]string) ([]mb.Event, error) { // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before if len(re.Query.Counters) == 0 { return nil, errors.New("no counters to read") @@ -98,7 +91,7 @@ func (re *Reader) Fetch(nameCounters map[string]string, processIdCounters map[st // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). // A flag is set if the second call has been executed else refresh will fail (reader.executed) if re.hasRun { - err := re.InitCounters(nameCounters, processIdCounters) + err := re.InitCounters(counters) if err != nil { return nil, errors.Wrap(err, "failed retrieving counters") } @@ -119,10 +112,12 @@ func (re *Reader) Fetch(nameCounters map[string]string, processIdCounters map[st for _, host := range re.Instances { events[host.Name] = mb.Event{ MetricSetFields: common.MapStr{ - "name": host.Name, - "worker_process_id": host.ProcessId, + "name": host.Name, }, } + if host.ProcessId != 0 { + events[host.Name].MetricSetFields.Put("worker_process_id", host.ProcessId) + } for counterPath, values := range values { for _, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. @@ -132,7 +127,7 @@ func (re *Reader) Fetch(nameCounters map[string]string, processIdCounters map[st "error", val.Err, logp.Namespace("website"), "query", counterPath) continue } - if val.Instance == host.Name || val.Instance == string(host.ProcessId) { + if val.Instance == host.Name || val.Instance == string(host.ProcessName) { events[host.Name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) } } @@ -151,3 +146,38 @@ func (re *Reader) Fetch(nameCounters map[string]string, processIdCounters map[st func (re *Reader) Close() error { return re.Query.Close() } + +func (re *Reader) GetInstances(instances []Instance) ([]Instance, error) { + var newQueries []string + counter := "\\Process(w3wp*)\\ID Process" + counters, err := re.Query.ExpandWildCardPath(counter) + for _, count := range counters { + if err := re.Query.AddCounter(count, "", "float", true); err != nil { + return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, count) + } + newQueries = append(newQueries, counter) + } + err = re.Query.RemoveUnusedCounters(newQueries) + if err != nil { + return nil, errors.Wrap(err, "failed removing unused counter values") + } + if err := re.Query.CollectData(); err != nil { + return nil, errors.Wrap(err, "failed collecting counter values") + } + // Get the values. + counterValues, err := re.Query.GetFormattedCounterValues() + if err != nil { + return nil, errors.Wrap(err, "failed retrieving counter formatted values") + } + for i, instance := range instances { + for _, value := range counterValues { + for _, counterValue := range value { + if instance.ProcessId == int(counterValue.Measurement.(float64)) { + instances[i].ProcessName = counterValue.Instance + } + } + + } + } + return instances, nil +} diff --git a/metricbeat/module/iis/website/_meta/data.json b/metricbeat/module/iis/website/_meta/data.json index 32874cf585bd..4762f926f8fd 100644 --- a/metricbeat/module/iis/website/_meta/data.json +++ b/metricbeat/module/iis/website/_meta/data.json @@ -1,19 +1,35 @@ { - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" - }, - "metricset":{ - "host":"localhost", - "module":"iis", - "name":"website", - "rtt":44269 - }, - "iis":{ - "website":{ - "example": "website" + "@timestamp" : "2020-02-05T12:47:14.651Z", + "metricset" : { + "name" : "website", + "period" : 10000 + }, + "service" : { + "type" : "iis" + }, + "iis" : { + "website" : { + "name" : "Default Web Site", + "network" : { + "total_bytes_received" : 23122, + "current_connections" : 0, + "service_uptime" : 1195668.0, + "maximum_connections" : 1, + "total_post_requests" : 0, + "total_bytes_sent" : 11350, + "total_connection_attempts" : 33, + "bytes_sent_per_sec" : 0, + "post_requests_per_sec" : 0, + "delete_requests_per_sec" : 0, + "get_requests_per_sec" : 0, + "total_delete_requests" : 0, + "bytes_received_per_sec" : 0, + "total_get_requests" : 39 + } + } + }, + "event" : { + "dataset" : "iis.website", + "module" : "iis" } - }, - "type":"metricsets" } diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go new file mode 100644 index 000000000000..ced8c3cdb4da --- /dev/null +++ b/metricbeat/module/iis/website/reader.go @@ -0,0 +1,234 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package website + +import ( + "github.com/StackExchange/wmi" + "github.com/elastic/beats/metricbeat/module/iis" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper/windows/pdh" + "github.com/elastic/beats/metricbeat/mb" +) + +// Reader will contain the config options +type Reader struct { + Query pdh.Query // PDH Query + Websites []Website // Mapping of counter path to key used for the label (e.g. processor.name) + log *logp.Logger // logger + hasRun bool // will check if the reader has run a first time + WorkerProcesses map[string]string +} + +type Website struct { + Name string + WorkerProcessIds []int + ApplicationName string + counters map[string]string +} + +type WorkerProcess struct { + ProcessId int + InstanceName string + counters map[string]string +} + +type Application struct { + ApplicationPool string + SiteName string +} +type Worker struct { + AppPoolName string + ProcessId int +} + +// NewReader creates a new instance of Reader. +func NewReader() (*Reader, error) { + var query pdh.Query + if err := query.Open(); err != nil { + return nil, err + } + reader := &Reader{ + Query: query, + log: logp.NewLogger("website"), + } + + return reader, nil +} + +func (re *Reader) InitCounters() error { + websites, err:= GetWebsites() + if err!= nil { + return err + } + re.Websites = websites + re.WorkerProcesses = make(map[string]string) + var newQueries []string + for i, instance := range re.Websites { + re.Websites[i].counters = make(map[string]string) + for key, value := range iis.WebsiteCounters { + value = strings.Replace(value, "*", instance.Name, 1) + if err := re.Query.AddCounter(value, "", "float", true); err != nil { + return errors.Wrapf(err, `failed to add counter (query="%v")`, value) + } + newQueries = append(newQueries, value) + re.Websites[i].counters[value] = key + } + } + for key, value := range iis.AppPoolCounters { + counters, err := re.Query.ExpandWildCardPath(value) + if err != nil { + return errors.Wrapf(err, `failed to expand counter path (query="%v")`, value) + } + for _, count := range counters { + if err = re.Query.AddCounter(count, "", "float", true); err != nil { + return errors.Wrapf(err, `failed to add counter (query="%v")`, count) + } + newQueries = append(newQueries, count) + re.WorkerProcesses[count] = key + } + } + err = re.Query.RemoveUnusedCounters(newQueries) + if err != nil { + return errors.Wrap(err, "failed removing unused counter values") + } + return nil +} + +// Read executes a query and returns those values in an event. +func (re *Reader) Fetch() ([]mb.Event, error) { + // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before + if len(re.Query.Counters) == 0 { + return nil, errors.New("no counters to read") + } + + // refresh performance counter list + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + // A flag is set if the second call has been executed else refresh will fail (reader.executed) + if re.hasRun { + err := re.InitCounters() + if err != nil { + return nil, errors.Wrap(err, "failed retrieving counters") + } + } + + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if err := re.Query.CollectData(); err != nil { + return nil, errors.Wrap(err, "failed querying counter values") + } + + // Get the values. + values, err := re.Query.GetFormattedCounterValues() + if err != nil { + return nil, errors.Wrap(err, "failed formatting counter values") + } + workers := getProcessIds(values) + events := make(map[string]mb.Event) + for _, host := range re.Websites { + events[host.Name] = mb.Event{ + MetricSetFields: common.MapStr{ + "name": host.Name, + "application_pool": host.ApplicationName, + }, + } + for counterPath, value := range values { + for _, val := range value { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.hasRun { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("website"), "query", counterPath) + continue + } + if val.Instance == host.Name { + events[host.Name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) + } else if hasWorkerProcess(val.Instance, workers, host.WorkerProcessIds) { + events[host.Name].MetricSetFields.Put(re.WorkerProcesses[counterPath], val.Measurement) + } + } + + } + } + + re.hasRun = true + results := make([]mb.Event, 0, len(events)) + for _, val := range events { + results = append(results, val) + } + return results, nil +} + +// Close will close the PDH query for now. +func (re *Reader) Close() error { + return re.Query.Close() +} + +func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess { + var workers []WorkerProcess + for key, values := range counterValues { + if strings.Contains(key, "\\ID Process") { + workers = append(workers, WorkerProcess{InstanceName: values[0].Instance, ProcessId: int(values[0].Measurement.(float64))}) + } + } + return workers +} + +func hasWorkerProcess(instance string, workers []WorkerProcess, pids []int) bool { + for _, worker := range workers { + if worker.InstanceName == instance { + for _, pid := range pids { + if pid == worker.ProcessId { + return true + } + } + } + } + return false +} + +func GetWebsites() ([]Website, error) { + var applications []Application + err := wmi.QueryNamespace("Select ApplicationPool, SiteName from Application", &applications, "root\\webadministration") + if err != nil { + // Don't return from this error since the name space might exist. + } + var workerProcesses []Worker + err = wmi.QueryNamespace("Select AppPoolName, ProcessId from WorkerProcess", &workerProcesses, "root\\webadministration") + if err != nil { + // Don't return from this error since the name space might exist. + } + var sites []Website + for _, application := range applications { + site := Website{Name: application.SiteName, ApplicationName: application.ApplicationPool} + for _, wp := range workerProcesses { + if wp.AppPoolName == application.ApplicationPool { + site.WorkerProcessIds = append(site.WorkerProcessIds, wp.ProcessId) + } + } + sites = append(sites, site) + } + return sites, nil +} diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index 3f99903bfb8d..fb5909c3b7c1 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -43,7 +43,7 @@ func init() { type MetricSet struct { mb.BaseMetricSet log *logp.Logger - reader *iis.Reader + reader *Reader } // Config for the iis website metricset. @@ -61,17 +61,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } // init reader object - reader, err := iis.NewReader() + reader, err := NewReader() if err != nil { return nil, err } - // get instances will retrieve the websites running in iis and also filter on the sites configured by users - instances, err := getInstances(reader, config.Sites) - if err != nil { - return nil, err - } - reader.Instances = instances - if err := reader.InitCounters(iis.WebsiteCounters, nil); err != nil { + + if err := reader.InitCounters(); err != nil { return nil, err } return &MetricSet{ @@ -89,12 +84,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err := m.Module().UnpackConfig(&config); err != nil { return nil } - instances, err := getInstances(m.reader, config.Sites) - if err != nil { - return err - } - m.reader.Instances = instances - events, err := m.reader.Fetch(iis.WebsiteCounters, nil) + events, err := m.reader.Fetch() if err != nil { return errors.Wrap(err, "failed reading counters") } diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml similarity index 75% rename from metricbeat/modules.d/iis.yml.disabled rename to metricbeat/modules.d/iis.yml index b8091f97cbd6..ca0fc8da8e1b 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml @@ -3,8 +3,6 @@ - module: iis metricsets: - - webserver - website - - application_pool enabled: true - period: 10s + period: 3s diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml.disabled similarity index 100% rename from metricbeat/modules.d/system.yml rename to metricbeat/modules.d/system.yml.disabled From 6aa632b78315d5d235d9d39564c6bc7ba1da7e27 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 10 Feb 2020 13:38:25 +0100 Subject: [PATCH 20/36] test --- metricbeat/module/iis/website/reader.go | 80 +++++++++++++++++-- .../modules.d/{iis.yml => iis.yml.disabled} | 0 .../{system.yml.disabled => system.yml} | 2 +- 3 files changed, 76 insertions(+), 6 deletions(-) rename metricbeat/modules.d/{iis.yml => iis.yml.disabled} (100%) rename metricbeat/modules.d/{system.yml.disabled => system.yml} (98%) diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go index ced8c3cdb4da..86abe677db0c 100644 --- a/metricbeat/module/iis/website/reader.go +++ b/metricbeat/module/iis/website/reader.go @@ -20,11 +20,15 @@ package website import ( + "encoding/xml" "github.com/StackExchange/wmi" "github.com/elastic/beats/metricbeat/module/iis" - "strings" - + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" "github.com/pkg/errors" + "io/ioutil" + "os" + "strings" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -78,8 +82,8 @@ func NewReader() (*Reader, error) { } func (re *Reader) InitCounters() error { - websites, err:= GetWebsites() - if err!= nil { + websites, err := GetWebsites1() + if err != nil { return err } re.Websites = websites @@ -150,7 +154,7 @@ func (re *Reader) Fetch() ([]mb.Event, error) { for _, host := range re.Websites { events[host.Name] = mb.Event{ MetricSetFields: common.MapStr{ - "name": host.Name, + "name": host.Name, "application_pool": host.ApplicationName, }, } @@ -232,3 +236,69 @@ func GetWebsites() ([]Website, error) { } return sites, nil } + +func GetWebsites1() ([]Website, error) { + path := os.Getenv("windir") + body, err := ioutil.ReadFile(path + "/system32/inetsrv/config/applicationHost.config") + if err != nil { + return nil, err + } + var config Configuration + err = xml.Unmarshal(body, &config) + if err != nil { + return nil, err + } + var websites []Website + for _, site := range config.SystemApplicationHost.Sites.Site { + if site.Name != "Default Web Site" { + websites = append(websites, Website{Name: site.Name, ApplicationName: site.Application.ApplicationPool}) + } + } + processes, err := sysinfo.Processes() + if err != nil { + return nil, err + } + var filt []types.Process + for _, pro := range processes { + info, err := pro.Info() + if err != nil { + return nil, err + } + if info.Name == "w3wp.exe" { + for i, si := range websites { + if getapp(info.Args) == si.ApplicationName { + websites[i].WorkerProcessIds = append(websites[i].WorkerProcessIds, info.PID) + } + } + filt = append(filt, pro) + } + } + return websites, nil +} + +func getapp(as []string) string { + for i, sd := range as { + if sd == "-ap" { + return as[i+1] + } + } + return "" + +} + +type Configuration struct { + SystemApplicationHost struct { + Sites struct { + Site []struct { + Text string `xml:",chardata"` + Name string `xml:"name,attr"` + ID string `xml:"id,attr"` + Application struct { + Text string `xml:",chardata"` + Path string `xml:"path,attr"` + ApplicationPool string `xml:"applicationPool,attr"` + } `xml:"application"` + } `xml:"site"` + } `xml:"sites"` + } `xml:"system.applicationHost"` +} diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml.disabled similarity index 100% rename from metricbeat/modules.d/iis.yml rename to metricbeat/modules.d/iis.yml.disabled diff --git a/metricbeat/modules.d/system.yml.disabled b/metricbeat/modules.d/system.yml similarity index 98% rename from metricbeat/modules.d/system.yml.disabled rename to metricbeat/modules.d/system.yml index 603dee8c0a45..7242cd864dcf 100644 --- a/metricbeat/modules.d/system.yml.disabled +++ b/metricbeat/modules.d/system.yml @@ -5,7 +5,7 @@ period: 10s metricsets: - cpu - - load + #- load - memory - network - process From ca3f5898b159f78a9a800cc5beb357d3d20bd897 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 11 Feb 2020 11:48:49 +0100 Subject: [PATCH 21/36] work on apppool --- .../iis/application_pool/application_pool.go | 42 +--------- metricbeat/module/iis/reader.go | 81 ++++++++++++++++++- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index f237346e5d5b..8b4c2dd97fca 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -44,12 +44,11 @@ type MetricSet struct { mb.BaseMetricSet log *logp.Logger reader *iis.Reader - instanceReader *iis.Reader } // Config for the iis website metricset. type Config struct { - Names []string `config:"name"` + Names []string `config:"app_pool_name"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -83,7 +82,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, log: logp.NewLogger("application pool"), reader: reader, - instanceReader: instanceReader, }, nil } @@ -115,44 +113,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } -// getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users -func getInstances(names []string, instanceReader *iis.Reader) ([]iis.Instance, error) { - processes, err := sysinfo.Processes() - if err != nil { - return nil, err - } - var w3processes []iis.Instance - for _, p := range processes { - info, err := p.Info() - if err != nil { - continue - } - if info.Name == "w3wp.exe" { - var app string - if len(info.Args) > 0 { - for i, ar := range info.Args { - if ar == "-ap" { - app = info.Args[i+1] - continue - } - } - } - w3processes = append(w3processes, iis.Instance{ProcessId: info.PID, Name: app}) - } - } - if len(names) == 0 { - return instanceReader.GetInstances(w3processes) - } - var filtered []iis.Instance - for _, n := range names { - for _, w3 := range w3processes { - if n == w3.Name { - filtered = append(filtered, w3) - } - } - } - return instanceReader.GetInstances(filtered) -} diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/reader.go index 5848cff84b01..976bc00ff1a0 100644 --- a/metricbeat/module/iis/reader.go +++ b/metricbeat/module/iis/reader.go @@ -20,6 +20,8 @@ package iis import ( + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" "strings" "github.com/pkg/errors" @@ -33,15 +35,14 @@ import ( // Reader will contain the config options type Reader struct { Query pdh.Query // PDH Query - Instances []Instance // Mapping of counter path to key used for the label (e.g. processor.name) + Instances []ApplicationPool // Mapping of counter path to key used for the label (e.g. processor.name) log *logp.Logger // logger hasRun bool // will check if the reader has run a first time } -type Instance struct { +type ApplicationPool struct { Name string - ProcessId int - ProcessName string + WorkerProcessIds []int counters map[string]string } @@ -181,3 +182,75 @@ func (re *Reader) GetInstances(instances []Instance) ([]Instance, error) { } return instances, nil } + +// getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users +func getApplicationPools(names []string) ([]ApplicationPool, error) { + processes, err := sysinfo.Processes() + if err != nil { + return nil, err + } + var w3processes []ApplicationPool + for _, p := range processes { + info, err := p.Info() + if err != nil { + continue + } + if info.Name == "w3wp.exe" { + var app string + if len(info.Args) > 0 { + for i, ar := range info.Args { + if ar == "-ap" && len(info.Args) > i+1 { + app = info.Args[i+1] + continue + } + } + } + + w3processes = append(w3processes, iis.Instance{ProcessId: info.PID, Name: app}) + } + } + if len(names) == 0 { + return instanceReader.GetInstances(w3processes) + } + var filtered []iis.Instance + for _, n := range names { + for _, w3 := range w3processes { + if n == w3.Name { + filtered = append(filtered, w3) + } + } + } + return instanceReader.GetInstances(filtered) +} + +func addAppPool(info types.ProcessInfo, appPools *[]ApplicationPool){ + if info.Name == "w3wp.exe" { + var app string + if len(info.Args) > 0 { + for i, ar := range info.Args { + if ar == "-ap" && len(info.Args) > i+1 { + app = info.Args[i+1] + continue + } + } + } +for i, app:= range appPools { + appPools[i].Name == app { + + } +} + w3processes = append(w3processes, iis.Instance{ProcessId: info.PID, Name: app}) + } +} + +func hasApp(appName string, appPools []ApplicationPool) bool { + for _, app:= range appPools { + if app.Name == appName{ +return true + + } + return false + } + +} + From b7398164fbc83a4d8b2f21c5f7f4360c49609df6 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 12 Feb 2020 11:28:43 +0100 Subject: [PATCH 22/36] work on app pool --- .../iis/application_pool/application_pool.go | 26 +-- .../iis/{ => application_pool}/reader.go | 186 ++++++++---------- metricbeat/module/iis/website/website.go | 39 ---- metricbeat/modules.d/iis.yml.disabled | 4 +- metricbeat/modules.d/system.yml | 2 +- 5 files changed, 95 insertions(+), 162 deletions(-) rename metricbeat/module/iis/{ => application_pool}/reader.go (59%) diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index 8b4c2dd97fca..31ed13310718 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -23,8 +23,6 @@ import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/iis" - "github.com/elastic/go-sysinfo" "github.com/pkg/errors" ) @@ -43,7 +41,7 @@ func init() { type MetricSet struct { mb.BaseMetricSet log *logp.Logger - reader *iis.Reader + reader *Reader } // Config for the iis website metricset. @@ -60,22 +58,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } // instantiate reader object - reader, err := iis.NewReader() - if err != nil { - return nil, err - } - // instantiate reader object that should retrieve the process ids and process names for the worker processes - instanceReader, err := iis.NewReader() + reader, err := NewReader() if err != nil { return nil, err } - instances, err := getInstances(config.Names, instanceReader) - if err != nil { - return nil, err - } - reader.Instances = instances - if err := reader.InitCounters(iis.AppPoolCounters); err != nil { + if err := reader.InitCounters(config.Names); err != nil { return nil, err } return &MetricSet{ @@ -93,12 +81,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err := m.Module().UnpackConfig(&config); err != nil { return nil } - instances, err := getInstances(config.Names, m.instanceReader) - if err != nil { - return err - } - m.reader.Instances = instances - events, err := m.reader.Fetch(iis.AppPoolCounters) + + events, err := m.reader.Fetch(config.Names) if err != nil { return errors.Wrap(err, "failed reading counters") } diff --git a/metricbeat/module/iis/reader.go b/metricbeat/module/iis/application_pool/reader.go similarity index 59% rename from metricbeat/module/iis/reader.go rename to metricbeat/module/iis/application_pool/reader.go index 976bc00ff1a0..dbd0f773a2f3 100644 --- a/metricbeat/module/iis/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -17,11 +17,11 @@ // +build windows -package iis +package application_pool import ( + "github.com/elastic/beats/metricbeat/module/iis" "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" "strings" "github.com/pkg/errors" @@ -34,16 +34,23 @@ import ( // Reader will contain the config options type Reader struct { - Query pdh.Query // PDH Query - Instances []ApplicationPool // Mapping of counter path to key used for the label (e.g. processor.name) - log *logp.Logger // logger - hasRun bool // will check if the reader has run a first time + Query pdh.Query // PDH Query + ApplicationPools []ApplicationPool // Mapping of counter path to key used for the label (e.g. processor.name) + log *logp.Logger // logger + hasRun bool // will check if the reader has run a first time + WorkerProcesses map[string]string } type ApplicationPool struct { - Name string - WorkerProcessIds []int - counters map[string]string + Name string + WorkerProcessIds []int + counters map[string]string +} + +type WorkerProcess struct { + ProcessId int + InstanceName string + counters map[string]string } // NewReader creates a new instance of Reader. @@ -60,20 +67,25 @@ func NewReader() (*Reader, error) { return reader, nil } -func (re *Reader) InitCounters(counters map[string]string) error { +func (re *Reader) InitCounters(filtered []string) error { + apps, err := getApplicationPools(filtered) + re.ApplicationPools = apps + re.WorkerProcesses = make(map[string]string) var newQueries []string - for i, instance := range re.Instances { - re.Instances[i].counters = make(map[string]string) - for key, value := range counters { - value = strings.Replace(value, "*", instance.Name, 1) - if err := re.Query.AddCounter(value, "", "float", true); err != nil { - return errors.Wrapf(err, `failed to add counter (query="%v")`, value) + for key, value := range iis.AppPoolCounters { + counters, err := re.Query.ExpandWildCardPath(value) + if err != nil { + return errors.Wrapf(err, `failed to expand counter path (query="%v")`, value) + } + for _, count := range counters { + if err = re.Query.AddCounter(count, "", "float", true); err != nil { + return errors.Wrapf(err, `failed to add counter (query="%v")`, count) } - newQueries = append(newQueries, value) - re.Instances[i].counters[value] = key + newQueries = append(newQueries, count) + re.WorkerProcesses[count] = key } } - err := re.Query.RemoveUnusedCounters(newQueries) + err = re.Query.RemoveUnusedCounters(newQueries) if err != nil { return errors.Wrap(err, "failed removing unused counter values") } @@ -81,7 +93,7 @@ func (re *Reader) InitCounters(counters map[string]string) error { } // Read executes a query and returns those values in an event. -func (re *Reader) Fetch(counters map[string]string) ([]mb.Event, error) { +func (re *Reader) Fetch(names []string) ([]mb.Event, error) { // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before if len(re.Query.Counters) == 0 { return nil, errors.New("no counters to read") @@ -92,7 +104,7 @@ func (re *Reader) Fetch(counters map[string]string) ([]mb.Event, error) { // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). // A flag is set if the second call has been executed else refresh will fail (reader.executed) if re.hasRun { - err := re.InitCounters(counters) + err := re.InitCounters(names) if err != nil { return nil, errors.Wrap(err, "failed retrieving counters") } @@ -109,18 +121,16 @@ func (re *Reader) Fetch(counters map[string]string) ([]mb.Event, error) { if err != nil { return nil, errors.Wrap(err, "failed formatting counter values") } + workers := getProcessIds(values) events := make(map[string]mb.Event) - for _, host := range re.Instances { - events[host.Name] = mb.Event{ + for _, appPool := range re.ApplicationPools { + events[appPool.Name] = mb.Event{ MetricSetFields: common.MapStr{ - "name": host.Name, + "name": appPool.Name, }, } - if host.ProcessId != 0 { - events[host.Name].MetricSetFields.Put("worker_process_id", host.ProcessId) - } - for counterPath, values := range values { - for _, val := range values { + for counterPath, value := range values { + for _, val := range value { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if val.Err != nil && !re.hasRun { @@ -128,10 +138,13 @@ func (re *Reader) Fetch(counters map[string]string) ([]mb.Event, error) { "error", val.Err, logp.Namespace("website"), "query", counterPath) continue } - if val.Instance == host.Name || val.Instance == string(host.ProcessName) { - events[host.Name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) + if val.Instance == appPool.Name { + events[appPool.Name].MetricSetFields.Put(appPool.counters[counterPath], val.Measurement) + } else if hasWorkerProcess(val.Instance, workers, appPool.WorkerProcessIds) { + events[appPool.Name].MetricSetFields.Put(re.WorkerProcesses[counterPath], val.Measurement) } } + } } @@ -148,109 +161,82 @@ func (re *Reader) Close() error { return re.Query.Close() } -func (re *Reader) GetInstances(instances []Instance) ([]Instance, error) { - var newQueries []string - counter := "\\Process(w3wp*)\\ID Process" - counters, err := re.Query.ExpandWildCardPath(counter) - for _, count := range counters { - if err := re.Query.AddCounter(count, "", "float", true); err != nil { - return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, count) - } - newQueries = append(newQueries, counter) - } - err = re.Query.RemoveUnusedCounters(newQueries) +// getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users +func getApplicationPools(names []string) ([]ApplicationPool, error) { + processes, err := getw3wpProceses() if err != nil { - return nil, errors.Wrap(err, "failed removing unused counter values") + return nil, err } - if err := re.Query.CollectData(); err != nil { - return nil, errors.Wrap(err, "failed collecting counter values") + var appPools = make(map[string][]int) + for key, value := range processes { + appPools[value] = append(appPools[value], key) + //if _, ok:= applicationPools[value]; ok { + // applicationPools[value] = append(applicationPools[value], key) + //} } - // Get the values. - counterValues, err := re.Query.GetFormattedCounterValues() - if err != nil { - return nil, errors.Wrap(err, "failed retrieving counter formatted values") + var applicationPools []ApplicationPool + for key, value := range appPools { + applicationPools = append(applicationPools, ApplicationPool{Name: key, WorkerProcessIds: value}) } - for i, instance := range instances { - for _, value := range counterValues { - for _, counterValue := range value { - if instance.ProcessId == int(counterValue.Measurement.(float64)) { - instances[i].ProcessName = counterValue.Instance - } - } + if len(names) == 0 { + return applicationPools, nil + } + var filtered []ApplicationPool + for _, n := range names { + for _, w3 := range applicationPools { + if n == w3.Name { + filtered = append(filtered, w3) + } } } - return instances, nil + return filtered, nil } -// getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users -func getApplicationPools(names []string) ([]ApplicationPool, error) { +func getw3wpProceses() (map[int]string, error) { processes, err := sysinfo.Processes() if err != nil { return nil, err } - var w3processes []ApplicationPool + wps := make(map[int]string) for _, p := range processes { info, err := p.Info() if err != nil { continue } if info.Name == "w3wp.exe" { - var app string if len(info.Args) > 0 { for i, ar := range info.Args { if ar == "-ap" && len(info.Args) > i+1 { - app = info.Args[i+1] + wps[info.PID] = info.Args[i+1] continue } } } - - w3processes = append(w3processes, iis.Instance{ProcessId: info.PID, Name: app}) - } - } - if len(names) == 0 { - return instanceReader.GetInstances(w3processes) - } - var filtered []iis.Instance - for _, n := range names { - for _, w3 := range w3processes { - if n == w3.Name { - filtered = append(filtered, w3) - } } } - return instanceReader.GetInstances(filtered) + return wps, nil } -func addAppPool(info types.ProcessInfo, appPools *[]ApplicationPool){ - if info.Name == "w3wp.exe" { - var app string - if len(info.Args) > 0 { - for i, ar := range info.Args { - if ar == "-ap" && len(info.Args) > i+1 { - app = info.Args[i+1] - continue - } - } +func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess { + var workers []WorkerProcess + for key, values := range counterValues { + if strings.Contains(key, "\\ID Process") { + workers = append(workers, WorkerProcess{InstanceName: values[0].Instance, ProcessId: int(values[0].Measurement.(float64))}) } -for i, app:= range appPools { - appPools[i].Name == app { - - } -} - w3processes = append(w3processes, iis.Instance{ProcessId: info.PID, Name: app}) } + return workers } -func hasApp(appName string, appPools []ApplicationPool) bool { - for _, app:= range appPools { - if app.Name == appName{ -return true - +func hasWorkerProcess(instance string, workers []WorkerProcess, pids []int) bool { + for _, worker := range workers { + if worker.InstanceName == instance { + for _, pid := range pids { + if pid == worker.ProcessId { + return true + } + } } - return false } - + return false } - diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go index fb5909c3b7c1..03b00718dd41 100644 --- a/metricbeat/module/iis/website/website.go +++ b/metricbeat/module/iis/website/website.go @@ -25,7 +25,6 @@ import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/module/iis" ) // init registers the MetricSet with the central registry as soon as the program @@ -99,41 +98,3 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } -// getInstances will retrieve the websites running in iis and also filter on the sites configured by users -func getInstances(reader *iis.Reader, sites []string) ([]iis.Instance, error) { - _, instances, err := reader.Query.GetCountersAndInstances("Web Service") - if err != nil { - reader.Query.Close() - return nil, err - } - return filterOnInstances(sites, instances), nil -} - -// filterOnInstances will filter on the sites configured by users -func filterOnInstances(hosts []string, instances []string) []iis.Instance { - filtered := make([]iis.Instance, 0) - // remove _Total and empty instances - for _, instance := range instances { - if instance == "_Total" || instance == "" { - continue - } - if containsHost(instance, hosts) { - filtered = append(filtered, iis.Instance{Name: instance}) - } - } - return filtered -} - -// containsHost checks if array contains site/hostname -func containsHost(item string, array []string) bool { - // if no hosts specified all instances are selected - if len(array) == 0 { - return true - } - for _, i := range array { - if i == item { - return true - } - } - return false -} diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml.disabled index ca0fc8da8e1b..b8091f97cbd6 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml.disabled @@ -3,6 +3,8 @@ - module: iis metricsets: + - webserver - website + - application_pool enabled: true - period: 3s + period: 10s diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index 7242cd864dcf..603dee8c0a45 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -5,7 +5,7 @@ period: 10s metricsets: - cpu - #- load + - load - memory - network - process From 552c1b75ab512a15b0c80f206ff741df025710a3 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 12 Feb 2020 13:30:38 +0100 Subject: [PATCH 23/36] work on website metricset --- metricbeat/docs/modules/iis/website.asciidoc | 1 + metricbeat/include/list_common.go | 1 - metricbeat/module/iis/module.yml | 1 + metricbeat/module/iis/website/manifest.yml | 54 ++++ metricbeat/module/iis/website/reader.go | 304 ------------------ metricbeat/module/iis/website/website.go | 100 ------ .../modules.d/{iis.yml.disabled => iis.yml} | 2 - .../{system.yml => system.yml.disabled} | 0 8 files changed, 56 insertions(+), 407 deletions(-) create mode 100644 metricbeat/module/iis/website/manifest.yml delete mode 100644 metricbeat/module/iis/website/reader.go delete mode 100644 metricbeat/module/iis/website/website.go rename metricbeat/modules.d/{iis.yml.disabled => iis.yml} (82%) rename metricbeat/modules.d/{system.yml => system.yml.disabled} (100%) diff --git a/metricbeat/docs/modules/iis/website.asciidoc b/metricbeat/docs/modules/iis/website.asciidoc index ccbf0416e6f3..1132d876ad49 100644 --- a/metricbeat/docs/modules/iis/website.asciidoc +++ b/metricbeat/docs/modules/iis/website.asciidoc @@ -9,6 +9,7 @@ beta[] include::../../../module/iis/website/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index abaccc0e4bb4..cda8e748363a 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -78,7 +78,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http/server" _ "github.com/elastic/beats/metricbeat/module/iis" _ "github.com/elastic/beats/metricbeat/module/iis/application_pool" - _ "github.com/elastic/beats/metricbeat/module/iis/website" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/module/iis/module.yml b/metricbeat/module/iis/module.yml index 5b0be61e8419..48b01480e3dd 100644 --- a/metricbeat/module/iis/module.yml +++ b/metricbeat/module/iis/module.yml @@ -1,3 +1,4 @@ name: iis metricsets: - webserver +- website diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml new file mode 100644 index 000000000000..d72e316b452d --- /dev/null +++ b/metricbeat/module/iis/website/manifest.yml @@ -0,0 +1,54 @@ +default: true +input: + module: windows + metricset: perfmon + processors: + - drop_event.when.regexp: + iis.website.name: '^/(_Total|cgroup)($|/)' + defaults: + perfmon.group_measurements_by_instance: true + perfmon.ignore_non_existent_counters: true + perfmon.counters: + #network + - instance_label: 'name' + measurement_label: network.total_bytes_received + query: '\Web Service(*)\Total Bytes Received' + - instance_label: 'name' + measurement_label: network.total_bytes_sent + query: '\Web Service(*)\Total Bytes Sent' + - instance_label: 'name' + measurement_label: network.bytes_sent_per_sec + query: '\Web Service(*)\Bytes Sent/sec' + - instance_label: 'name' + measurement_label: network.bytes_received_per_sec + query: '\Web Service(*)\Bytes Received/sec' + - instance_label: 'name' + measurement_label: network.current_connections + query: 'Web Service(*)\Current Connections' + - instance_label: 'name' + measurement_label: network.maximum_connections + query: '\Web Service(*)\Maximum Connections' + - instance_label: 'name' + measurement_label: network.total_connection_attempts + query: '\Web Service(*)\Total Connection Attempts (all instances)' + - instance_label: 'name' + measurement_label: network.total_get_requests + query: '\Web Service(*)\Total Get Requests' + - instance_label: 'name' + measurement_label: network.get_requests_per_sec + query: '\Web Service(*)\Get Requests/sec' + - instance_label: 'name' + measurement_label: network.total_post_requests + query: '\Web Service(*)\Total Post Requests' + - instance_label: 'name' + measurement_label: network.post_requests_per_sec + query: '\Web Service(*)\Post Requests/sec' + - instance_label: 'name' + measurement_label: network.total_delete_requests + query: '\Web Service(*)\Total Delete Requests' + - instance_label: 'name' + measurement_label: network.delete_requests_per_sec + query: '\Web Service(*)\Delete Requests/sec' + - instance_label: 'name' + measurement_label: network.service_uptime + query: '\Web Service(*)\Service Uptime' diff --git a/metricbeat/module/iis/website/reader.go b/metricbeat/module/iis/website/reader.go deleted file mode 100644 index 86abe677db0c..000000000000 --- a/metricbeat/module/iis/website/reader.go +++ /dev/null @@ -1,304 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build windows - -package website - -import ( - "encoding/xml" - "github.com/StackExchange/wmi" - "github.com/elastic/beats/metricbeat/module/iis" - "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" - "github.com/pkg/errors" - "io/ioutil" - "os" - "strings" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/helper/windows/pdh" - "github.com/elastic/beats/metricbeat/mb" -) - -// Reader will contain the config options -type Reader struct { - Query pdh.Query // PDH Query - Websites []Website // Mapping of counter path to key used for the label (e.g. processor.name) - log *logp.Logger // logger - hasRun bool // will check if the reader has run a first time - WorkerProcesses map[string]string -} - -type Website struct { - Name string - WorkerProcessIds []int - ApplicationName string - counters map[string]string -} - -type WorkerProcess struct { - ProcessId int - InstanceName string - counters map[string]string -} - -type Application struct { - ApplicationPool string - SiteName string -} -type Worker struct { - AppPoolName string - ProcessId int -} - -// NewReader creates a new instance of Reader. -func NewReader() (*Reader, error) { - var query pdh.Query - if err := query.Open(); err != nil { - return nil, err - } - reader := &Reader{ - Query: query, - log: logp.NewLogger("website"), - } - - return reader, nil -} - -func (re *Reader) InitCounters() error { - websites, err := GetWebsites1() - if err != nil { - return err - } - re.Websites = websites - re.WorkerProcesses = make(map[string]string) - var newQueries []string - for i, instance := range re.Websites { - re.Websites[i].counters = make(map[string]string) - for key, value := range iis.WebsiteCounters { - value = strings.Replace(value, "*", instance.Name, 1) - if err := re.Query.AddCounter(value, "", "float", true); err != nil { - return errors.Wrapf(err, `failed to add counter (query="%v")`, value) - } - newQueries = append(newQueries, value) - re.Websites[i].counters[value] = key - } - } - for key, value := range iis.AppPoolCounters { - counters, err := re.Query.ExpandWildCardPath(value) - if err != nil { - return errors.Wrapf(err, `failed to expand counter path (query="%v")`, value) - } - for _, count := range counters { - if err = re.Query.AddCounter(count, "", "float", true); err != nil { - return errors.Wrapf(err, `failed to add counter (query="%v")`, count) - } - newQueries = append(newQueries, count) - re.WorkerProcesses[count] = key - } - } - err = re.Query.RemoveUnusedCounters(newQueries) - if err != nil { - return errors.Wrap(err, "failed removing unused counter values") - } - return nil -} - -// Read executes a query and returns those values in an event. -func (re *Reader) Fetch() ([]mb.Event, error) { - // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before - if len(re.Query.Counters) == 0 { - return nil, errors.New("no counters to read") - } - - // refresh performance counter list - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - // A flag is set if the second call has been executed else refresh will fail (reader.executed) - if re.hasRun { - err := re.InitCounters() - if err != nil { - return nil, errors.Wrap(err, "failed retrieving counters") - } - } - - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if err := re.Query.CollectData(); err != nil { - return nil, errors.Wrap(err, "failed querying counter values") - } - - // Get the values. - values, err := re.Query.GetFormattedCounterValues() - if err != nil { - return nil, errors.Wrap(err, "failed formatting counter values") - } - workers := getProcessIds(values) - events := make(map[string]mb.Event) - for _, host := range re.Websites { - events[host.Name] = mb.Event{ - MetricSetFields: common.MapStr{ - "name": host.Name, - "application_pool": host.ApplicationName, - }, - } - for counterPath, value := range values { - for _, val := range value { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !re.hasRun { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("website"), "query", counterPath) - continue - } - if val.Instance == host.Name { - events[host.Name].MetricSetFields.Put(host.counters[counterPath], val.Measurement) - } else if hasWorkerProcess(val.Instance, workers, host.WorkerProcessIds) { - events[host.Name].MetricSetFields.Put(re.WorkerProcesses[counterPath], val.Measurement) - } - } - - } - } - - re.hasRun = true - results := make([]mb.Event, 0, len(events)) - for _, val := range events { - results = append(results, val) - } - return results, nil -} - -// Close will close the PDH query for now. -func (re *Reader) Close() error { - return re.Query.Close() -} - -func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess { - var workers []WorkerProcess - for key, values := range counterValues { - if strings.Contains(key, "\\ID Process") { - workers = append(workers, WorkerProcess{InstanceName: values[0].Instance, ProcessId: int(values[0].Measurement.(float64))}) - } - } - return workers -} - -func hasWorkerProcess(instance string, workers []WorkerProcess, pids []int) bool { - for _, worker := range workers { - if worker.InstanceName == instance { - for _, pid := range pids { - if pid == worker.ProcessId { - return true - } - } - } - } - return false -} - -func GetWebsites() ([]Website, error) { - var applications []Application - err := wmi.QueryNamespace("Select ApplicationPool, SiteName from Application", &applications, "root\\webadministration") - if err != nil { - // Don't return from this error since the name space might exist. - } - var workerProcesses []Worker - err = wmi.QueryNamespace("Select AppPoolName, ProcessId from WorkerProcess", &workerProcesses, "root\\webadministration") - if err != nil { - // Don't return from this error since the name space might exist. - } - var sites []Website - for _, application := range applications { - site := Website{Name: application.SiteName, ApplicationName: application.ApplicationPool} - for _, wp := range workerProcesses { - if wp.AppPoolName == application.ApplicationPool { - site.WorkerProcessIds = append(site.WorkerProcessIds, wp.ProcessId) - } - } - sites = append(sites, site) - } - return sites, nil -} - -func GetWebsites1() ([]Website, error) { - path := os.Getenv("windir") - body, err := ioutil.ReadFile(path + "/system32/inetsrv/config/applicationHost.config") - if err != nil { - return nil, err - } - var config Configuration - err = xml.Unmarshal(body, &config) - if err != nil { - return nil, err - } - var websites []Website - for _, site := range config.SystemApplicationHost.Sites.Site { - if site.Name != "Default Web Site" { - websites = append(websites, Website{Name: site.Name, ApplicationName: site.Application.ApplicationPool}) - } - } - processes, err := sysinfo.Processes() - if err != nil { - return nil, err - } - var filt []types.Process - for _, pro := range processes { - info, err := pro.Info() - if err != nil { - return nil, err - } - if info.Name == "w3wp.exe" { - for i, si := range websites { - if getapp(info.Args) == si.ApplicationName { - websites[i].WorkerProcessIds = append(websites[i].WorkerProcessIds, info.PID) - } - } - filt = append(filt, pro) - } - } - return websites, nil -} - -func getapp(as []string) string { - for i, sd := range as { - if sd == "-ap" { - return as[i+1] - } - } - return "" - -} - -type Configuration struct { - SystemApplicationHost struct { - Sites struct { - Site []struct { - Text string `xml:",chardata"` - Name string `xml:"name,attr"` - ID string `xml:"id,attr"` - Application struct { - Text string `xml:",chardata"` - Path string `xml:"path,attr"` - ApplicationPool string `xml:"applicationPool,attr"` - } `xml:"application"` - } `xml:"site"` - } `xml:"sites"` - } `xml:"system.applicationHost"` -} diff --git a/metricbeat/module/iis/website/website.go b/metricbeat/module/iis/website/website.go deleted file mode 100644 index 03b00718dd41..000000000000 --- a/metricbeat/module/iis/website/website.go +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build windows - -package website - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" -) - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.AddMetricSet("iis", "website", New) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - mb.BaseMetricSet - log *logp.Logger - reader *Reader -} - -// Config for the iis website metricset. -type Config struct { - Sites []string `config:"site"` -} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The website metricset is beta") - - var config Config - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - // init reader object - reader, err := NewReader() - if err != nil { - return nil, err - } - - if err := reader.InitCounters(); err != nil { - return nil, err - } - return &MetricSet{ - BaseMetricSet: base, - log: logp.NewLogger("website"), - reader: reader, - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right -// format. It publishes the event which is then forwarded to the output. In case -// of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(report mb.ReporterV2) error { - var config Config - if err := m.Module().UnpackConfig(&config); err != nil { - return nil - } - events, err := m.reader.Fetch() - if err != nil { - return errors.Wrap(err, "failed reading counters") - } - - for _, event := range events { - isOpen := report.Event(event) - if !isOpen { - break - } - } - - return nil -} - diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml similarity index 82% rename from metricbeat/modules.d/iis.yml.disabled rename to metricbeat/modules.d/iis.yml index b8091f97cbd6..c978e7adff7a 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml @@ -3,8 +3,6 @@ - module: iis metricsets: - - webserver - website - - application_pool enabled: true period: 10s diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml.disabled similarity index 100% rename from metricbeat/modules.d/system.yml rename to metricbeat/modules.d/system.yml.disabled From fadb63aad8b95c6e2feab050a8a4db2e2ed48dd2 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 13 Feb 2020 16:19:51 +0100 Subject: [PATCH 24/36] work on tests --- metricbeat/docs/fields.asciidoc | 30 ++++++-- metricbeat/docs/modules/iis.asciidoc | 34 +++++++++ metricbeat/module/iis/_meta/docs.asciidoc | 34 +++++++++ .../iis/application_pool/_meta/data.json | 35 +++++---- .../iis/application_pool/_meta/docs.asciidoc | 13 ++++ .../iis/application_pool/_meta/fields.yml | 8 ++- .../iis/application_pool/application_pool.go | 13 ++-- .../application_pool/application_pool_test.go | 25 +++++++ .../module/iis/application_pool/reader.go | 70 ++++++++++++------ .../iis/application_pool/reader_test.go | 71 +++++++++++++++++++ metricbeat/module/iis/counters.go | 55 -------------- metricbeat/module/iis/fields.go | 2 +- .../module/iis/webserver/_meta/data.json | 66 +++++++++++++---- .../module/iis/webserver/_meta/docs.asciidoc | 19 +++++ .../module/iis/webserver/_meta/fields.yml | 9 ++- metricbeat/module/iis/webserver/doc.go | 1 + metricbeat/module/iis/webserver/manifest.yml | 2 +- .../webserver/webserver_integration_test.go | 52 ++++++++++++++ .../module/iis/webserver/webserver_test.go | 13 ++++ .../module/iis/website/_meta/docs.asciidoc | 14 ++++ .../module/iis/website/_meta/fields.yml | 8 ++- metricbeat/module/iis/website/manifest.yml | 35 ++++----- .../iis/website/website_integration_test.go | 52 ++++++++++++++ metricbeat/module/iis/website/website_test.go | 13 ++++ metricbeat/module/windows/perfmon/perfmon.go | 8 +-- metricbeat/module/windows/perfmon/reader.go | 5 +- .../module/windows/perfmon/reader_test.go | 4 +- .../modules.d/{iis.yml => iis.yml.disabled} | 2 + .../{system.yml.disabled => system.yml} | 0 29 files changed, 537 insertions(+), 156 deletions(-) create mode 100644 metricbeat/module/iis/application_pool/reader_test.go delete mode 100644 metricbeat/module/iis/counters.go create mode 100644 metricbeat/module/iis/webserver/doc.go create mode 100644 metricbeat/module/iis/webserver/webserver_integration_test.go create mode 100644 metricbeat/module/iis/website/website_integration_test.go rename metricbeat/modules.d/{iis.yml => iis.yml.disabled} (82%) rename metricbeat/modules.d/{system.yml.disabled => system.yml} (100%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 23bb576024ef..d2a606431b7e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16835,28 +16835,46 @@ application_pool -*`iis.application_pool.example`*:: +*`iis.application_pool.name`*:: + -- -Example field +application pool name type: keyword -- -[float] -=== webserver +*`iis.application_pool.worker_processor_id`*:: ++ +-- +worker processor id + +type: keyword + +-- + +*`iis.webserver.*.*`*:: ++ +-- webserver -[float] -=== website +type: object +-- + +*`iis.website.*`*:: ++ +-- website +type: object + +-- + [[exported-fields-istio]] == istio fields diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 244bf8946796..6e1a0264de2b 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -9,6 +9,40 @@ beta[] This is the iis module. +IIS (Internet Information Services) is a secure, reliable, and scalable Web server that provides an easy to manage platform for developing and hosting Web applications and services. + +The `iis` module will periodically retrieve IIS related metrics using performance counters such as: + + - System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process). + - IIS performance counters like Web Service: Bytes Received/Sec, Web Service: Bytes Sent/Sec, etc, which are helpful to track to identify potential spikes in traffic. + - Web Service Cache counters in order to monitor user mode cache and output cache. + + +The `iis` module mericsets are `webserver`, `website` and `application_pool`. + +[float] +== Metricsets + +[float] +=== `webserver` +This metricset allows users to retrieve aggregated metrics for the entire webserver. + +[float] +=== `website` +This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered. + +[float] +=== `application_pool` +This metricset will collect metrics of specific application pools, users can configure which websites they want to monitor, else, all are considered. + + +[float] +=== Module-specific configuration notes + +`website.name`:: []string, users can specify the sites they would like to monitor. + +`application_pool.name`:: []string, users can specify the application pools they would like to monitor. + [float] diff --git a/metricbeat/module/iis/_meta/docs.asciidoc b/metricbeat/module/iis/_meta/docs.asciidoc index a59df72134d5..b92130651b37 100644 --- a/metricbeat/module/iis/_meta/docs.asciidoc +++ b/metricbeat/module/iis/_meta/docs.asciidoc @@ -1,2 +1,36 @@ This is the iis module. +IIS (Internet Information Services) is a secure, reliable, and scalable Web server that provides an easy to manage platform for developing and hosting Web applications and services. + +The `iis` module will periodically retrieve IIS related metrics using performance counters such as: + + - System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process). + - IIS performance counters like Web Service: Bytes Received/Sec, Web Service: Bytes Sent/Sec, etc, which are helpful to track to identify potential spikes in traffic. + - Web Service Cache counters in order to monitor user mode cache and output cache. + + +The `iis` module mericsets are `webserver`, `website` and `application_pool`. + +[float] +== Metricsets + +[float] +=== `webserver` +This metricset allows users to retrieve aggregated metrics for the entire webserver. + +[float] +=== `website` +This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered. + +[float] +=== `application_pool` +This metricset will collect metrics of specific application pools, users can configure which websites they want to monitor, else, all are considered. + + +[float] +=== Module-specific configuration notes + +`website.name`:: []string, users can specify the sites they would like to monitor. + +`application_pool.name`:: []string, users can specify the application pools they would like to monitor. + diff --git a/metricbeat/module/iis/application_pool/_meta/data.json b/metricbeat/module/iis/application_pool/_meta/data.json index befe502b30db..60e93f1ce0c4 100644 --- a/metricbeat/module/iis/application_pool/_meta/data.json +++ b/metricbeat/module/iis/application_pool/_meta/data.json @@ -1,19 +1,26 @@ { - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" + "@timestamp" : "2020-02-13T14:55:27.000Z", + "metricset" : { + "name" : "application_pool", + "period" : 10000 }, - "metricset":{ - "host":"localhost", - "module":"iis", - "name":"application_pool", - "rtt":44269 + "service" : { + "type" : "iis" }, - "iis":{ - "application_pool":{ - "example": "application_pool" + "iis" : { + "application_pool" : { + "net_clr" : { + "total_exceptions_thrown" : 0 + }, + "worker_process_id" : 11956, + "name" : "test.local", + "process" : { + "thread_count" : 32, + "private_byte" : 3.4684928E7, + "handle_count" : 743, + "virtual_bytes" : 6.49535488E8, + "working_set" : 4.2012672E7 + } } - }, - "type":"metricsets" + } } diff --git a/metricbeat/module/iis/application_pool/_meta/docs.asciidoc b/metricbeat/module/iis/application_pool/_meta/docs.asciidoc index e8cd5b6d8413..e85da0ee7e89 100644 --- a/metricbeat/module/iis/application_pool/_meta/docs.asciidoc +++ b/metricbeat/module/iis/application_pool/_meta/docs.asciidoc @@ -1 +1,14 @@ This is the application_pool metricset of the module iis. + +This metricset allows users to retrieve relevant metrics for the application pools running on IIS. + +Metric values are divided in several groups: + +The `process` object contains System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process). + +The `net_clr` object which returns ASP.NET error rate counter values. +Users can specify the application pools they would like to monitor using the configuration option `application_pool.name`, else, all application pools are considered. + + + + diff --git a/metricbeat/module/iis/application_pool/_meta/fields.yml b/metricbeat/module/iis/application_pool/_meta/fields.yml index 3eec06cddaef..10a5e4d0632c 100644 --- a/metricbeat/module/iis/application_pool/_meta/fields.yml +++ b/metricbeat/module/iis/application_pool/_meta/fields.yml @@ -4,7 +4,11 @@ description: > application_pool fields: - - name: example + - name: name type: keyword description: > - Example field + application pool name + - name: worker_processor_id + type: keyword + description: > + worker processor id diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index 31ed13310718..a369a9c552fc 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -46,7 +46,7 @@ type MetricSet struct { // Config for the iis website metricset. type Config struct { - Names []string `config:"app_pool_name"` + Names []string `config:"application_pool.name"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -58,12 +58,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } // instantiate reader object - reader, err := NewReader() + reader, err := newReader() if err != nil { return nil, err } - if err := reader.InitCounters(config.Names); err != nil { + if err := reader.initCounters(config.Names); err != nil { return nil, err } return &MetricSet{ @@ -82,7 +82,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } - events, err := m.reader.Fetch(config.Names) + events, err := m.reader.fetch(config.Names) if err != nil { return errors.Wrap(err, "failed reading counters") } @@ -93,10 +93,5 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { break } } - return nil } - - - - diff --git a/metricbeat/module/iis/application_pool/application_pool_test.go b/metricbeat/module/iis/application_pool/application_pool_test.go index 3bc21201b3e7..9411123936ab 100644 --- a/metricbeat/module/iis/application_pool/application_pool_test.go +++ b/metricbeat/module/iis/application_pool/application_pool_test.go @@ -15,4 +15,29 @@ // specific language governing permissions and limitations // under the License. +// +build integration +// +build windows + package application_pool + +import ( + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "testing" + "time" +) + +func TestMetricsetNoErrors(t *testing.T) { + config := map[string]interface{}{ + "module": "iis", + "metricsets": []string{"application_pool"}, + } + + ms := mbtest.NewReportingMetricSetV2Error(t, config) + mbtest.ReportingFetchV2Error(ms) + time.Sleep(60 * time.Millisecond) + + _, errs := mbtest.ReportingFetchV2Error(ms) + if len(errs) > 0 { + t.Fatal(errs) + } +} diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index dbd0f773a2f3..5e6177cba838 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -20,7 +20,6 @@ package application_pool import ( - "github.com/elastic/beats/metricbeat/module/iis" "github.com/elastic/go-sysinfo" "strings" @@ -32,7 +31,7 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -// Reader will contain the config options +// Reader strucr will contain the pdh query and config options type Reader struct { Query pdh.Query // PDH Query ApplicationPools []ApplicationPool // Mapping of counter path to key used for the label (e.g. processor.name) @@ -41,20 +40,39 @@ type Reader struct { WorkerProcesses map[string]string } +// ApplicationPool struct contains the list of applications and their worker processes type ApplicationPool struct { Name string WorkerProcessIds []int counters map[string]string } +// WorkerProcess struct contains the worker process details type WorkerProcess struct { ProcessId int InstanceName string - counters map[string]string } -// NewReader creates a new instance of Reader. -func NewReader() (*Reader, error) { +var appPoolCounters = map[string]string{ + "worker_process_id": "\\Process(w3wp*)\\ID Process", + "process.cpu_usage_perc": "\\Process(w3wp*)\\% Processor Time", + "process.handle_count": "\\Process(w3wp*)\\Handle Count", + "process.thread_count": "\\Process(w3wp*)\\Thread Count", + "process.working_set": "\\Process(w3wp*)\\Working Set", + "process.private_byte": "\\Process(w3wp*)\\Private Bytes", + "process.virtual_bytes": "\\Process(w3wp*)\\Virtual Bytes", + "process.page_faults_per_sec": "\\Process(w3wp*)\\Page Faults/sec", + "process.io_read_operations_per_sec": "\\Process(w3wp*)\\IO Read Operations/sec", + "process.io_write_operations_per_sec": "\\Process(w3wp*)\\IO Write Operations/sec", + "net_clr.total_exceptions_thrown": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown", + "net_clr.exceptions_thrown_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown / sec", + "net_clr.filters_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Filters / sec", + "net_clr.finallys_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Finallys / sec", + "net_clr.throw_to_catch_depth_per_sec": "\\.NET CLR Exceptions(w3wp*)\\Throw To Catch Depth / sec", +} + +// newReader creates a new instance of Reader. +func newReader() (*Reader, error) { var query pdh.Query if err := query.Open(); err != nil { return nil, err @@ -67,12 +85,20 @@ func NewReader() (*Reader, error) { return reader, nil } -func (re *Reader) InitCounters(filtered []string) error { +// initCounters func retrieves the running application worker processes and adds the counters to the pdh query +func (re *Reader) initCounters(filtered []string) error { apps, err := getApplicationPools(filtered) + if err != nil { + return errors.Wrap(err, "failed retrieving running worker processes") + } + if len(apps) == 0 { + re.log.Info("no running application pools found") + return nil + } re.ApplicationPools = apps re.WorkerProcesses = make(map[string]string) var newQueries []string - for key, value := range iis.AppPoolCounters { + for key, value := range appPoolCounters { counters, err := re.Query.ExpandWildCardPath(value) if err != nil { return errors.Wrapf(err, `failed to expand counter path (query="%v")`, value) @@ -92,24 +118,22 @@ func (re *Reader) InitCounters(filtered []string) error { return nil } -// Read executes a query and returns those values in an event. -func (re *Reader) Fetch(names []string) ([]mb.Event, error) { - // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before - if len(re.Query.Counters) == 0 { - return nil, errors.New("no counters to read") - } - +// fetch executes collects the query data and maps the counter values to events. +func (re *Reader) fetch(names []string) ([]mb.Event, error) { // refresh performance counter list // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). // A flag is set if the second call has been executed else refresh will fail (reader.executed) - if re.hasRun { - err := re.InitCounters(names) + if re.hasRun || len(re.Query.Counters) == 0 { + err := re.initCounters(names) if err != nil { return nil, errors.Wrap(err, "failed retrieving counters") } } - + // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before + if len(re.Query.Counters) == 0 { + return nil, nil + } // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if err := re.Query.CollectData(); err != nil { @@ -121,6 +145,7 @@ func (re *Reader) Fetch(names []string) ([]mb.Event, error) { if err != nil { return nil, errors.Wrap(err, "failed formatting counter values") } + workers := getProcessIds(values) events := make(map[string]mb.Event) for _, appPool := range re.ApplicationPools { @@ -157,11 +182,11 @@ func (re *Reader) Fetch(names []string) ([]mb.Event, error) { } // Close will close the PDH query for now. -func (re *Reader) Close() error { +func (re *Reader) close() error { return re.Query.Close() } -// getInstances method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users +// getApplicationPools method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users func getApplicationPools(names []string) ([]ApplicationPool, error) { processes, err := getw3wpProceses() if err != nil { @@ -170,15 +195,11 @@ func getApplicationPools(names []string) ([]ApplicationPool, error) { var appPools = make(map[string][]int) for key, value := range processes { appPools[value] = append(appPools[value], key) - //if _, ok:= applicationPools[value]; ok { - // applicationPools[value] = append(applicationPools[value], key) - //} } var applicationPools []ApplicationPool for key, value := range appPools { applicationPools = append(applicationPools, ApplicationPool{Name: key, WorkerProcessIds: value}) } - if len(names) == 0 { return applicationPools, nil } @@ -193,6 +214,7 @@ func getApplicationPools(names []string) ([]ApplicationPool, error) { return filtered, nil } +// getw3wpProceses func retrieves the running w3wp process ids func getw3wpProceses() (map[int]string, error) { processes, err := sysinfo.Processes() if err != nil { @@ -218,6 +240,7 @@ func getw3wpProceses() (map[int]string, error) { return wps, nil } +// getProcessIds func maps the process ids from the counter values to worker process obj func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess { var workers []WorkerProcess for key, values := range counterValues { @@ -228,6 +251,7 @@ func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess return workers } +// hasWorkerProcess func checks if workerprocess list contains the process id func hasWorkerProcess(instance string, workers []WorkerProcess, pids []int) bool { for _, worker := range workers { if worker.InstanceName == instance { diff --git a/metricbeat/module/iis/application_pool/reader_test.go b/metricbeat/module/iis/application_pool/reader_test.go new file mode 100644 index 000000000000..5362b795f4ed --- /dev/null +++ b/metricbeat/module/iis/application_pool/reader_test.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package application_pool + +import ( + "github.com/elastic/beats/metricbeat/helper/windows/pdh" + "github.com/stretchr/testify/assert" + "testing" +) + +// TestNewReaderValid should successfully instantiate the reader. +func TestNewReaderValid(t *testing.T) { + reader, err := newReader() + assert.Nil(t, err) + assert.NotNil(t, reader) + assert.NotNil(t, reader.Query) + assert.NotNil(t, reader.Query.Handle) + assert.NotNil(t, reader.Query.Counters) + assert.Zero(t, len(reader.Query.Counters)) + defer reader.close() +} + +// TestInitCounters should successfully instantiate the reader counters. +func TestInitCounters(t *testing.T) { + reader, err := newReader() + assert.NotNil(t, reader) + assert.Nil(t, err) + + err = reader.initCounters([]string{}) + assert.Nil(t, err) + // if iis is not enabled, the reader.ApplicationPools is empty + if len(reader.ApplicationPools) > 0 { + assert.NotZero(t, len(reader.Query.Counters)) + assert.NotZero(t, len(reader.WorkerProcesses)) + } + defer reader.close() +} + +func TestGetProcessIds(t *testing.T) { + var key = "\\Process(w3wp#1)\\ID Process" + var counters = []pdh.CounterValue{ + { + Instance: "w3wp#1", + Measurement: 124.00, + Err: nil, + }, + } + counterList := make(map[string][]pdh.CounterValue) + counterList[key] = counters + workerProcesses := getProcessIds(counterList) + assert.NotZero(t, len(workerProcesses)) + assert.Equal(t, float64(workerProcesses[0].ProcessId), counters[0].Measurement.(float64)) + assert.Equal(t, workerProcesses[0].InstanceName, counters[0].Instance) +} diff --git a/metricbeat/module/iis/counters.go b/metricbeat/module/iis/counters.go deleted file mode 100644 index 5d2efe530410..000000000000 --- a/metricbeat/module/iis/counters.go +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package iis - -var ( - WebsiteCounters = map[string]string{ - "network.total_bytes_received": "\\Web Service(*)\\Total Bytes Received", - "network.total_bytes_sent": "\\Web Service(*)\\Total Bytes Sent", - "network.bytes_sent_per_sec": "\\Web Service(*)\\Bytes Sent/sec", - "network.bytes_received_per_sec": "\\Web Service(*)\\Bytes Received/sec", - "network.current_connections": "\\Web Service(*)\\Current Connections", - "network.maximum_connections": "\\Web Service(*)\\Maximum Connections", - "network.total_connection_attempts": "\\Web Service(*)\\Total Connection Attempts (all instances)", - "network.total_get_requests": "\\Web Service(*)\\Total Get Requests", - "network.get_requests_per_sec": "\\Web Service(*)\\Get Requests/sec", - "network.total_post_requests": "\\Web Service(*)\\Total Post Requests", - "network.post_requests_per_sec": "\\Web Service(*)\\Post Requests/sec", - "network.total_delete_requests": "\\Web Service(*)\\Total Delete Requests", - "network.delete_requests_per_sec": "\\Web Service(*)\\Delete Requests/sec", - "network.service_uptime": "\\Web Service(*)\\Service Uptime", - } - AppPoolCounters = map[string]string{ - "worker_process_id": "\\Process(w3wp*)\\ID Process", - "process.cpu_usage_perc": "\\Process(w3wp*)\\% Processor Time", - "process.handle_count": "\\Process(w3wp*)\\Handle Count", - "process.thread_count": "\\Process(w3wp*)\\Thread Count", - "process.working_set": "\\Process(w3wp*)\\Working Set", - "process.private_byte": "\\Process(w3wp*)\\Private Bytes", - "process.virtual_bytes": "\\Process(w3wp*)\\Virtual Bytes", - "process.page_faults_per_sec": "\\Process(w3wp*)\\Page Faults/sec", - "process.io_read_operations_per_sec": "\\Process(w3wp*)\\IO Read Operations/sec", - "process.io_write_operations_per_sec": "\\Process(w3wp*)\\IO Write Operations/sec", - "net_clr.total_exceptions_thrown": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown", - "net_clr.exceptions_thrown_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown / sec", - "net_clr.filters_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Filters / sec", - "net_clr.finallys_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Finallys / sec", - "net_clr.throw_to_catch_depth_per_sec": "\\.NET CLR Exceptions(w3wp*)\\Throw To Catch Depth / sec", - //"asp_net_applications.request_bytes_total": "\\ASP.NET Applications(w3wp*)\\Request Bytes In Total", - } -) diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index 877c832d515d..75f9403095ed 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -32,5 +32,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/iis. func AssetIis() string { - return "eJykkD1uxCAUhHtOMdp+L0CRLueI2GUSPRkbBDiObx9h8oMd5BQ7JTx932iuGLhqiCQFZMmOGheRdFGAZbpHCVn8pPGkAJQ7jN7OjgqIdDSJGjdmoxTwKnQ26e3yismM/CaX5DVQ4y36OXy9dAR7SAsyITi5m3L8Erx3Pwc9csm+3u9711pz4jjWaqvxw4xhm6RNbTVwXXy0h7+TDiXPFVilf5ZYeEuM74z/TnCi6UH6i7VayXxUukf0leozAAD//xsprTU=" + return "eJzMksFuszAQhO9+ihFHpOQBfPhfBRnYRPtjWGvtNOLtK5dAgNIoUi/dA4fZ0cy3wAkdjRbM0QCJkyeLgjkWBmgpNsohsQwW/wyA7EMv7c2TAZQ8uUgWNSVnDHBh8m20X84TBtfTnJwnjYEsriq38FAOCrYh6yAXgufGZXMVRPxiOErOs8V76oet07zo2GOt0fJzs5iROhrvou1u9wJgB4EM8T1+7r2LdqRVUGkoRtGK91W/wJjCsYRjFb4AUB1JP0jP5bl849VPOFL/pyat5Emopu3Fi/thWfUuBB6uD2dRFu991gVzUQ8v4UR//QpOzz/BfAYAAP//6u79+A==" } diff --git a/metricbeat/module/iis/webserver/_meta/data.json b/metricbeat/module/iis/webserver/_meta/data.json index 49f09a5610e1..e01bfc68c05c 100644 --- a/metricbeat/module/iis/webserver/_meta/data.json +++ b/metricbeat/module/iis/webserver/_meta/data.json @@ -1,19 +1,57 @@ { - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" + "@timestamp" : "2020-02-12T15:40:39.727Z", + "service" : { + "type" : "iis" }, - "metricset":{ - "host":"localhost", - "module":"iis", - "name":"webserver", - "rtt":44269 - }, - "iis":{ - "webserver":{ - "example": "webserver" + "iis" : { + "webserver" : { + "process" : { + "worker_process_count" : 4, + "virtual_bytes" : 2.699976704E9, + "io_read_operations_per_sec" : 0, + "page_faults_per_Sec" : 0, + "private_byte" : 6.5712128E7, + "working_set" : 1396736.0, + "handle_count" : 0, + "cpu_usage_perc" : 0, + "io_write_operations_per_sec" : 0, + "thread_count" : 0 + }, + "network" : { + "total_delete_requests" : 0, + "bytes_received_per_sec" : 0, + "total_post_requests" : 0, + "delete_requests_per_sec" : 0, + "total_connection_attempts" : 73, + "bytes_sent_per_sec" : 0, + "get_requests_per_sec" : 0, + "maximum_connections" : 11, + "total_bytes_received" : 68170, + "service_uptime" : 588930, + "total_get_requests" : 162, + "total_bytes_sent" : 2898527.0, + "post_requests_per_sec" : 0 + }, + "cache" : { + "total_uris_cached" : 24, + "uri_cache_misses" : 161, + "file_cache_hits" : 21, + "maximum_file_cache_memory_usage" : 696, + "output_cache_current_memory_usage" : 0, + "total_files_cached" : 4, + "current_file_cache_memory_usage" : 0, + "uri_cache_hits" : 21, + "file_cache_misses" : 45, + "current_files_cached" : 0, + "output_cache_total_hits" : 0, + "output_cache_current_items" : 0, + "output_cache_total_misses" : 182, + "current_uris_cached" : 0 + } } }, - "type":"metricsets" + "metricset" : { + "name" : "webserver", + "period" : 10000 + } } diff --git a/metricbeat/module/iis/webserver/_meta/docs.asciidoc b/metricbeat/module/iis/webserver/_meta/docs.asciidoc index 8c0c2ea43b10..41e1658e075d 100644 --- a/metricbeat/module/iis/webserver/_meta/docs.asciidoc +++ b/metricbeat/module/iis/webserver/_meta/docs.asciidoc @@ -1 +1,20 @@ This is the webserver metricset of the module iis. + + +This metricset allows users to retrieve relevant metrics from IIS. + +Metric values are divided in several groups: + +The `process` object contains System/Process counters like the the overall server and CPU usage for the IIS Worker Processes and memory (currently used and available memory for the IIS Worker Processes). + +The `network` object contains the IIS Performance counters like: +Web Service: Bytes Received/Sec (helpful to track to identify potential spikes in traffic), Web Service: Bytes Sent/Sec (helpful to track to identify potential spikes in traffic), +Web Service: Current Connections (through experience with their apps app, users can identify what is a normal value for this) and others. + +The `cache` object contains metrics from the user mode cache and output cache. + + + + + + diff --git a/metricbeat/module/iis/webserver/_meta/fields.yml b/metricbeat/module/iis/webserver/_meta/fields.yml index 89041c173a82..b31375bdbb21 100644 --- a/metricbeat/module/iis/webserver/_meta/fields.yml +++ b/metricbeat/module/iis/webserver/_meta/fields.yml @@ -1,5 +1,8 @@ -- name: webserver - type: group +- name: webserver.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" description: > webserver - release: beta + diff --git a/metricbeat/module/iis/webserver/doc.go b/metricbeat/module/iis/webserver/doc.go new file mode 100644 index 000000000000..c5303c20adae --- /dev/null +++ b/metricbeat/module/iis/webserver/doc.go @@ -0,0 +1 @@ +package webserver diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 23d16696891a..841d4f71fc34 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -5,7 +5,7 @@ input: defaults: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true - perfmon.group_all_counter: true + perfmon.group_all_counter: "worker_process_count" perfmon.counters: #network - instance_label: '' diff --git a/metricbeat/module/iis/webserver/webserver_integration_test.go b/metricbeat/module/iis/webserver/webserver_integration_test.go new file mode 100644 index 000000000000..fc859125d39c --- /dev/null +++ b/metricbeat/module/iis/webserver/webserver_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package webserver + +import ( + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" + "testing" + // Register input module and metricset + _ "github.com/elastic/beats/metricbeat/module/windows" + _ "github.com/elastic/beats/metricbeat/module/windows/perfmon" +) + +func TestData(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + m.WriteEvents(t, "") +} + +func TestFetch(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + events, errs := m.FetchEvents() + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "iis", + "metricsets": []string{"webserver"}, + } +} diff --git a/metricbeat/module/iis/webserver/webserver_test.go b/metricbeat/module/iis/webserver/webserver_test.go index b772274c9236..dc3bf8852620 100644 --- a/metricbeat/module/iis/webserver/webserver_test.go +++ b/metricbeat/module/iis/webserver/webserver_test.go @@ -15,4 +15,17 @@ // specific language governing permissions and limitations // under the License. +// +build windows + package webserver + +import ( + "github.com/elastic/beats/metricbeat/mb" + "os" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/metricbeat/module/iis/website/_meta/docs.asciidoc b/metricbeat/module/iis/website/_meta/docs.asciidoc index b2ea5dab2da3..9f09d47fef9d 100644 --- a/metricbeat/module/iis/website/_meta/docs.asciidoc +++ b/metricbeat/module/iis/website/_meta/docs.asciidoc @@ -1 +1,15 @@ This is the website metricset of the module iis. + +This metricset allows users to retrieve relevant metrics for the websites running on IIS. + +The metrics contain the IIS Performance counter values like: + Web Service: Bytes Received/Sec (helpful to track to identify potential spikes in traffic), Web Service: Bytes Sent/Sec (helpful to track to identify potential spikes in traffic), + Web Service: Current Connections (through experience with their apps app, users can identify what is a normal value for this) and others. + + +Users can specify the sites they would like to monitor using the configuration option `website.name`, else, all websites are considered. + + + + + diff --git a/metricbeat/module/iis/website/_meta/fields.yml b/metricbeat/module/iis/website/_meta/fields.yml index 726d14ae6382..e463c0d9dccb 100644 --- a/metricbeat/module/iis/website/_meta/fields.yml +++ b/metricbeat/module/iis/website/_meta/fields.yml @@ -1,6 +1,8 @@ -- name: website - type: group +- name: website.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" description: > website - release: beta diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index d72e316b452d..e0fe3bf47374 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -2,53 +2,54 @@ default: true input: module: windows metricset: perfmon - processors: - - drop_event.when.regexp: - iis.website.name: '^/(_Total|cgroup)($|/)' defaults: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true perfmon.counters: #network - instance_label: 'name' - measurement_label: network.total_bytes_received + measurement_label: total_bytes_received query: '\Web Service(*)\Total Bytes Received' - instance_label: 'name' - measurement_label: network.total_bytes_sent + measurement_label: total_bytes_sent query: '\Web Service(*)\Total Bytes Sent' - instance_label: 'name' - measurement_label: network.bytes_sent_per_sec + measurement_label: bytes_sent_per_sec query: '\Web Service(*)\Bytes Sent/sec' - instance_label: 'name' - measurement_label: network.bytes_received_per_sec + measurement_label: bytes_received_per_sec query: '\Web Service(*)\Bytes Received/sec' - instance_label: 'name' - measurement_label: network.current_connections + measurement_label: current_connections query: 'Web Service(*)\Current Connections' - instance_label: 'name' - measurement_label: network.maximum_connections + measurement_label: maximum_connections query: '\Web Service(*)\Maximum Connections' - instance_label: 'name' - measurement_label: network.total_connection_attempts + measurement_label: total_connection_attempts query: '\Web Service(*)\Total Connection Attempts (all instances)' - instance_label: 'name' - measurement_label: network.total_get_requests + measurement_label: total_get_requests query: '\Web Service(*)\Total Get Requests' - instance_label: 'name' - measurement_label: network.get_requests_per_sec + measurement_label: get_requests_per_sec query: '\Web Service(*)\Get Requests/sec' - instance_label: 'name' - measurement_label: network.total_post_requests + measurement_label: total_post_requests query: '\Web Service(*)\Total Post Requests' - instance_label: 'name' - measurement_label: network.post_requests_per_sec + measurement_label: post_requests_per_sec query: '\Web Service(*)\Post Requests/sec' - instance_label: 'name' - measurement_label: network.total_delete_requests + measurement_label: total_delete_requests query: '\Web Service(*)\Total Delete Requests' - instance_label: 'name' - measurement_label: network.delete_requests_per_sec + measurement_label: delete_requests_per_sec query: '\Web Service(*)\Delete Requests/sec' - instance_label: 'name' - measurement_label: network.service_uptime + measurement_label: service_uptime query: '\Web Service(*)\Service Uptime' +processors: +- drop_event.when.equals: + iis.website.name: '_Total' + diff --git a/metricbeat/module/iis/website/website_integration_test.go b/metricbeat/module/iis/website/website_integration_test.go new file mode 100644 index 000000000000..e1194d3ba3e2 --- /dev/null +++ b/metricbeat/module/iis/website/website_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package website + +import ( + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" + "testing" + // Register input module and metricset + _ "github.com/elastic/beats/metricbeat/module/windows" + _ "github.com/elastic/beats/metricbeat/module/windows/perfmon" +) + +func TestData(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + m.WriteEvents(t, "") +} + +func TestFetch(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + events, errs := m.FetchEvents() + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "iis", + "metricsets": []string{"website"}, + } +} diff --git a/metricbeat/module/iis/website/website_test.go b/metricbeat/module/iis/website/website_test.go index 81e02fe557b4..b9a8520c818e 100644 --- a/metricbeat/module/iis/website/website_test.go +++ b/metricbeat/module/iis/website/website_test.go @@ -15,4 +15,17 @@ // specific language governing permissions and limitations // under the License. +// +build windows + package website + +import ( + "github.com/elastic/beats/metricbeat/mb" + "os" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 44443b407b73..7a716e499853 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -40,10 +40,10 @@ type CounterConfig struct { // Config for the windows perfmon metricset. type Config struct { - IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` - GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` - CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCounters bool `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment + IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` + GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` + CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` + GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment } const metricsetName = "perfmon" diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 640f38e4a4ac..82c75234d857 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -20,6 +20,7 @@ package perfmon import ( + "fmt" "regexp" "strconv" "strings" @@ -149,7 +150,7 @@ func (this *Reader) Read(metricset string) ([]mb.Event, error) { return nil, errors.Wrap(err, "failed formatting counter values") } var events []mb.Event - if this.config.GroupAllCounters && (metricsetName != metricset) { + if this.config.GroupAllCountersTo != "" && (metricsetName != metricset) { event := this.groupToEvent(values) events = append(events, event) } else { @@ -242,7 +243,7 @@ func (this *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Even if val == 1 { continue } else { - event.MetricSetFields.Put(strings.Split(key, ".")[0]+".instances", val) + event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], this.config.GroupAllCountersTo), val) } } else { event.MetricSetFields.Put(key, val) diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index dd5a58aa5225..ebcd66cc35b3 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -73,11 +73,11 @@ func TestReadSuccessfully(t *testing.T) { } //Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we call reader.Read() twice. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - events, err := reader.Read() + events, err := reader.Read("perfmon") assert.Nil(t, err) assert.NotNil(t, events) assert.Zero(t, len(events)) - events, err = reader.Read() + events, err = reader.Read("perfmon") assert.Nil(t, err) assert.NotNil(t, events) assert.NotZero(t, len(events)) diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml.disabled similarity index 82% rename from metricbeat/modules.d/iis.yml rename to metricbeat/modules.d/iis.yml.disabled index c978e7adff7a..b8091f97cbd6 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml.disabled @@ -3,6 +3,8 @@ - module: iis metricsets: + - webserver - website + - application_pool enabled: true period: 10s diff --git a/metricbeat/modules.d/system.yml.disabled b/metricbeat/modules.d/system.yml similarity index 100% rename from metricbeat/modules.d/system.yml.disabled rename to metricbeat/modules.d/system.yml From 706b3932f39457950f4c77c1a15fbf5cc94a2e8c Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 14 Feb 2020 11:59:57 +0100 Subject: [PATCH 25/36] Work on website --- metricbeat/docs/fields.asciidoc | 13 ++++++-- metricbeat/docs/modules/iis.asciidoc | 5 ++++ metricbeat/metricbeat.reference.yml | 5 ++++ metricbeat/module/iis/_meta/config.yml | 5 ++++ metricbeat/module/iis/fields.go | 2 +- metricbeat/module/iis/webserver/doc.go | 17 +++++++++++ metricbeat/module/iis/website/_meta/data.json | 30 +++++++++---------- .../module/iis/website/_meta/fields.yml | 12 ++++---- metricbeat/module/iis/website/manifest.yml | 5 +++- metricbeat/modules.d/iis.yml.disabled | 5 ++++ 10 files changed, 73 insertions(+), 26 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d2a606431b7e..a81900e48159 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16865,13 +16865,20 @@ type: object -- -*`iis.website.*`*:: +[float] +=== website + +website + + + +*`iis.website.name`*:: + -- -website +website name -type: object +type: keyword -- diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 6e1a0264de2b..7ffe9ad8b8fe 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -61,6 +61,11 @@ metricbeat.modules: - application_pool enabled: true period: 10s + + # filter on application pool names + # application_pool.name: [] + # filter on site names + # website.name: [] ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 02d5cb07e88c..690571ccbe2d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -355,6 +355,11 @@ metricbeat.modules: enabled: true period: 10s + # filter on application pool names + # application_pool.name: [] + # filter on site names + # website.name: [] + #------------------------------- Jolokia Module ------------------------------- - module: jolokia #metricsets: ["jmx"] diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml index 7fff9dc3388a..c467d4163462 100644 --- a/metricbeat/module/iis/_meta/config.yml +++ b/metricbeat/module/iis/_meta/config.yml @@ -5,3 +5,8 @@ - application_pool enabled: true period: 10s + + # filter on application pool names + # application_pool.name: [] + # filter on site names + # website.name: [] diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index 75f9403095ed..07a039e383dc 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -32,5 +32,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/iis. func AssetIis() string { - return "eJzMksFuszAQhO9+ihFHpOQBfPhfBRnYRPtjWGvtNOLtK5dAgNIoUi/dA4fZ0cy3wAkdjRbM0QCJkyeLgjkWBmgpNsohsQwW/wyA7EMv7c2TAZQ8uUgWNSVnDHBh8m20X84TBtfTnJwnjYEsriq38FAOCrYh6yAXgufGZXMVRPxiOErOs8V76oet07zo2GOt0fJzs5iROhrvou1u9wJgB4EM8T1+7r2LdqRVUGkoRtGK91W/wJjCsYRjFb4AUB1JP0jP5bl849VPOFL/pyat5Emopu3Fi/thWfUuBB6uD2dRFu991gVzUQ8v4UR//QpOzz/BfAYAAP//6u79+A==" + return "eJy0kktuwzAMRPc6xcDLAMkBtOhVDMVmAjayKUhKA9++UGW7/iNAUS68IAczzyLPeFCnwRwUEDla0iiYQ6GAmkLl2UWWVuNDAUg6NFI/LSnAkyUTSONK0SgF3JhsHfSP8ozWNDQ4p4qdI427l6frOxsBc5OpkXHOcmWSuHQidhRsOaea4/32N1NzHWQssaZo6TsbDEgP6l7i68XsAGABgQSxth9yX+If5EvnpaIQxJe8jPoDRjbHaI6J+QhA10D+i/zldDm98fQZR66fVMVJOzfKPL1ZMTvDsjHOcXvvlcWpeG+tI+bY3fwTjvQfR7W2Xt/S7iXtL/BwfX1m9vsOAAD//5CFBRA=" } diff --git a/metricbeat/module/iis/webserver/doc.go b/metricbeat/module/iis/webserver/doc.go index c5303c20adae..b772274c9236 100644 --- a/metricbeat/module/iis/webserver/doc.go +++ b/metricbeat/module/iis/webserver/doc.go @@ -1 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package webserver diff --git a/metricbeat/module/iis/website/_meta/data.json b/metricbeat/module/iis/website/_meta/data.json index 4762f926f8fd..b4bbb9c25ba9 100644 --- a/metricbeat/module/iis/website/_meta/data.json +++ b/metricbeat/module/iis/website/_meta/data.json @@ -10,22 +10,20 @@ "iis" : { "website" : { "name" : "Default Web Site", - "network" : { - "total_bytes_received" : 23122, - "current_connections" : 0, - "service_uptime" : 1195668.0, - "maximum_connections" : 1, - "total_post_requests" : 0, - "total_bytes_sent" : 11350, - "total_connection_attempts" : 33, - "bytes_sent_per_sec" : 0, - "post_requests_per_sec" : 0, - "delete_requests_per_sec" : 0, - "get_requests_per_sec" : 0, - "total_delete_requests" : 0, - "bytes_received_per_sec" : 0, - "total_get_requests" : 39 - } + "total_bytes_received" : 23122, + "current_connections" : 0, + "service_uptime" : 1195668.0, + "maximum_connections" : 1, + "total_post_requests" : 0, + "total_bytes_sent" : 11350, + "total_connection_attempts" : 33, + "bytes_sent_per_sec" : 0, + "post_requests_per_sec" : 0, + "delete_requests_per_sec" : 0, + "get_requests_per_sec" : 0, + "total_delete_requests" : 0, + "bytes_received_per_sec" : 0, + "total_get_requests" : 39 } }, "event" : { diff --git a/metricbeat/module/iis/website/_meta/fields.yml b/metricbeat/module/iis/website/_meta/fields.yml index e463c0d9dccb..74be20459303 100644 --- a/metricbeat/module/iis/website/_meta/fields.yml +++ b/metricbeat/module/iis/website/_meta/fields.yml @@ -1,8 +1,10 @@ -- name: website.* +- name: website + type: group release: beta - type: object - object_type: float - object_type_mapping_type: "*" description: > website - + fields: + - name: name + type: keyword + description: > + website name diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index e0fe3bf47374..cc15eea164e7 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -3,6 +3,9 @@ input: module: windows metricset: perfmon defaults: + website_params: + - field: names + option: website.name perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true perfmon.counters: @@ -51,5 +54,5 @@ input: query: '\Web Service(*)\Service Uptime' processors: - drop_event.when.equals: - iis.website.name: '_Total' + iis.website.name: '_Total' diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml.disabled index b8091f97cbd6..85b3253451ac 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml.disabled @@ -8,3 +8,8 @@ - application_pool enabled: true period: 10s + + # filter on application pool names + # application_pool.name: [] + # filter on site names + # website.name: [] From a1656be2266578a1196549d440e1dea0467d0419 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 14 Feb 2020 12:48:04 +0100 Subject: [PATCH 26/36] work on website --- libbeat/logp/core.go | 4 +-- metricbeat/module/iis/website/manifest.yml | 4 +-- metricbeat/module/windows/perfmon/perfmon.go | 3 +- metricbeat/module/windows/perfmon/reader.go | 35 +++++++++++++++++-- .../modules.d/{iis.yml.disabled => iis.yml} | 4 +-- 5 files changed, 38 insertions(+), 12 deletions(-) rename metricbeat/modules.d/{iis.yml.disabled => iis.yml} (81%) diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index 5dd990714f41..902c02fefe55 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -38,7 +38,7 @@ import ( var ( _log unsafe.Pointer // Pointer to a coreLogger. Access via atomic.LoadPointer. - _defaultGoLog = golog.Writer() + //_defaultGoLog = golog.Writer() ) func init() { @@ -108,7 +108,7 @@ func Configure(cfg Config) error { _, stdlogEnabled := selectors["stdlog"] _, allEnabled := selectors["*"] if stdlogEnabled || allEnabled { - golog.SetOutput(_defaultGoLog) + //golog.SetOutput(_defaultGoLog) } sink = selectiveWrapper(sink, selectors) diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index cc15eea164e7..744f88cd1b8f 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -8,6 +8,7 @@ input: option: website.name perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true + perfmon.filter_by_instance: {{.names}} perfmon.counters: #network - instance_label: 'name' @@ -52,7 +53,4 @@ input: - instance_label: 'name' measurement_label: service_uptime query: '\Web Service(*)\Service Uptime' -processors: -- drop_event.when.equals: - iis.website.name: '_Total' diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 7a716e499853..4b2282777b22 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -43,7 +43,8 @@ type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment + GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment + FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment } const metricsetName = "perfmon" diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 82c75234d857..a9eb11c5f73f 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -150,9 +150,14 @@ func (this *Reader) Read(metricset string) ([]mb.Event, error) { return nil, errors.Wrap(err, "failed formatting counter values") } var events []mb.Event - if this.config.GroupAllCountersTo != "" && (metricsetName != metricset) { - event := this.groupToEvent(values) - events = append(events, event) + if metricsetName != metricset { + if len(this.config.FilterByInstance) > 0 { + values = filterValuesByInstances(this.config.FilterByInstance, values) + } + if this.config.GroupAllCountersTo != "" { + event := this.groupToEvent(values) + events = append(events, event) + } } else { events = this.groupToEvents(values) } @@ -265,3 +270,27 @@ func matchesParentProcess(instanceName string) (bool, string) { } return false, instanceName } + +// filterValuesByInstances func filters the counter valus based on the instances provided by the user, this is applied to the 'website' metricset at the moment so the _Total instance should be filtered out +func filterValuesByInstances(instances []string, counterValues map[string][]pdh.CounterValue) map[string][]pdh.CounterValue { + filteredValues := make(map[string][]pdh.CounterValue) + for key, values := range counterValues { + filteredValues[key] = []pdh.CounterValue{} + for _, value := range values { + if containsInstance(value, instances) && value.Instance != "_Total" { + filteredValues[key] = append(filteredValues[key], value) + } + } + } + return filteredValues +} + +// containsInstance func checks if the counter value contains a filtered instance +func containsInstance(counterValue pdh.CounterValue, array []string) bool { + for _, ins := range array { + if ins == counterValue.Instance { + return true + } + } + return false +} diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml similarity index 81% rename from metricbeat/modules.d/iis.yml.disabled rename to metricbeat/modules.d/iis.yml index 85b3253451ac..c1b38f55292d 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml @@ -3,13 +3,11 @@ - module: iis metricsets: - - webserver - website - - application_pool enabled: true period: 10s # filter on application pool names # application_pool.name: [] # filter on site names - # website.name: [] + website.name: ["test.local"] From 1d760b14ff23ae1156f3485382e1814bf0ade3b0 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 14 Feb 2020 16:06:11 +0100 Subject: [PATCH 27/36] work on website --- libbeat/logp/core.go | 4 ++-- metricbeat/docs/modules/iis.asciidoc | 16 ++++++++++++++++ metricbeat/module/iis/_meta/docs.asciidoc | 16 ++++++++++++++++ .../iis/application_pool/application_pool.go | 13 +++++++------ .../application_pool/application_pool_test.go | 3 ++- metricbeat/module/iis/application_pool/reader.go | 3 ++- .../module/iis/application_pool/reader_test.go | 6 ++++-- .../iis/webserver/webserver_integration_test.go | 7 +++++-- .../module/iis/webserver/webserver_test.go | 3 ++- metricbeat/module/iis/website/manifest.yml | 5 +---- .../iis/website/website_integration_test.go | 7 +++++-- metricbeat/module/iis/website/website_test.go | 3 ++- metricbeat/module/windows/perfmon/perfmon.go | 4 ++-- .../modules.d/{iis.yml => iis.yml.disabled} | 4 +++- 14 files changed, 69 insertions(+), 25 deletions(-) rename metricbeat/modules.d/{iis.yml => iis.yml.disabled} (81%) diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index 902c02fefe55..5dd990714f41 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -38,7 +38,7 @@ import ( var ( _log unsafe.Pointer // Pointer to a coreLogger. Access via atomic.LoadPointer. - //_defaultGoLog = golog.Writer() + _defaultGoLog = golog.Writer() ) func init() { @@ -108,7 +108,7 @@ func Configure(cfg Config) error { _, stdlogEnabled := selectors["stdlog"] _, allEnabled := selectors["*"] if stdlogEnabled || allEnabled { - //golog.SetOutput(_defaultGoLog) + golog.SetOutput(_defaultGoLog) } sink = selectiveWrapper(sink, selectors) diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 7ffe9ad8b8fe..ca587f1d7e51 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -20,6 +20,22 @@ The `iis` module will periodically retrieve IIS related metrics using performanc The `iis` module mericsets are `webserver`, `website` and `application_pool`. +[source,yaml] +---- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] + # filter on site names + # website.name: [] +---- + [float] == Metricsets diff --git a/metricbeat/module/iis/_meta/docs.asciidoc b/metricbeat/module/iis/_meta/docs.asciidoc index b92130651b37..dba0b21ed538 100644 --- a/metricbeat/module/iis/_meta/docs.asciidoc +++ b/metricbeat/module/iis/_meta/docs.asciidoc @@ -11,6 +11,22 @@ The `iis` module will periodically retrieve IIS related metrics using performanc The `iis` module mericsets are `webserver`, `website` and `application_pool`. +[source,yaml] +---- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] + # filter on site names + # website.name: [] +---- + [float] == Metricsets diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index a369a9c552fc..835f05c15564 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -20,10 +20,11 @@ package application_pool import ( + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" - "github.com/pkg/errors" ) // init registers the MetricSet with the central registry as soon as the program @@ -40,8 +41,8 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - log *logp.Logger - reader *Reader + log *logp.Logger + reader *Reader } // Config for the iis website metricset. @@ -67,9 +68,9 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } return &MetricSet{ - BaseMetricSet: base, - log: logp.NewLogger("application pool"), - reader: reader, + BaseMetricSet: base, + log: logp.NewLogger("application pool"), + reader: reader, }, nil } diff --git a/metricbeat/module/iis/application_pool/application_pool_test.go b/metricbeat/module/iis/application_pool/application_pool_test.go index 9411123936ab..f118d135c1f2 100644 --- a/metricbeat/module/iis/application_pool/application_pool_test.go +++ b/metricbeat/module/iis/application_pool/application_pool_test.go @@ -21,9 +21,10 @@ package application_pool import ( - mbtest "github.com/elastic/beats/metricbeat/mb/testing" "testing" "time" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestMetricsetNoErrors(t *testing.T) { diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index 5e6177cba838..c0f2970cb7ae 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -20,9 +20,10 @@ package application_pool import ( - "github.com/elastic/go-sysinfo" "strings" + "github.com/elastic/go-sysinfo" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" diff --git a/metricbeat/module/iis/application_pool/reader_test.go b/metricbeat/module/iis/application_pool/reader_test.go index 5362b795f4ed..3fddd416dd98 100644 --- a/metricbeat/module/iis/application_pool/reader_test.go +++ b/metricbeat/module/iis/application_pool/reader_test.go @@ -20,9 +20,11 @@ package application_pool import ( - "github.com/elastic/beats/metricbeat/helper/windows/pdh" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/metricbeat/helper/windows/pdh" ) // TestNewReaderValid should successfully instantiate the reader. diff --git a/metricbeat/module/iis/webserver/webserver_integration_test.go b/metricbeat/module/iis/webserver/webserver_integration_test.go index fc859125d39c..643b883bd835 100644 --- a/metricbeat/module/iis/webserver/webserver_integration_test.go +++ b/metricbeat/module/iis/webserver/webserver_integration_test.go @@ -21,9 +21,12 @@ package webserver import ( - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + // Register input module and metricset _ "github.com/elastic/beats/metricbeat/module/windows" _ "github.com/elastic/beats/metricbeat/module/windows/perfmon" diff --git a/metricbeat/module/iis/webserver/webserver_test.go b/metricbeat/module/iis/webserver/webserver_test.go index dc3bf8852620..4d639aadbe55 100644 --- a/metricbeat/module/iis/webserver/webserver_test.go +++ b/metricbeat/module/iis/webserver/webserver_test.go @@ -20,8 +20,9 @@ package webserver import ( - "github.com/elastic/beats/metricbeat/mb" "os" + + "github.com/elastic/beats/metricbeat/mb" ) func init() { diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index 744f88cd1b8f..8c92c28c636a 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -3,12 +3,9 @@ input: module: windows metricset: perfmon defaults: - website_params: - - field: names - option: website.name perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true - perfmon.filter_by_instance: {{.names}} + perfmon.filter_by_instance: [""] perfmon.counters: #network - instance_label: 'name' diff --git a/metricbeat/module/iis/website/website_integration_test.go b/metricbeat/module/iis/website/website_integration_test.go index e1194d3ba3e2..3b8bea6a5f37 100644 --- a/metricbeat/module/iis/website/website_integration_test.go +++ b/metricbeat/module/iis/website/website_integration_test.go @@ -21,9 +21,12 @@ package website import ( - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + // Register input module and metricset _ "github.com/elastic/beats/metricbeat/module/windows" _ "github.com/elastic/beats/metricbeat/module/windows/perfmon" diff --git a/metricbeat/module/iis/website/website_test.go b/metricbeat/module/iis/website/website_test.go index b9a8520c818e..f9618fa3bfbb 100644 --- a/metricbeat/module/iis/website/website_test.go +++ b/metricbeat/module/iis/website/website_test.go @@ -20,8 +20,9 @@ package website import ( - "github.com/elastic/beats/metricbeat/mb" "os" + + "github.com/elastic/beats/metricbeat/mb" ) func init() { diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 4b2282777b22..70b0ec1402ff 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -43,8 +43,8 @@ type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment - FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment + GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment + FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment } const metricsetName = "perfmon" diff --git a/metricbeat/modules.d/iis.yml b/metricbeat/modules.d/iis.yml.disabled similarity index 81% rename from metricbeat/modules.d/iis.yml rename to metricbeat/modules.d/iis.yml.disabled index c1b38f55292d..85b3253451ac 100644 --- a/metricbeat/modules.d/iis.yml +++ b/metricbeat/modules.d/iis.yml.disabled @@ -3,11 +3,13 @@ - module: iis metricsets: + - webserver - website + - application_pool enabled: true period: 10s # filter on application pool names # application_pool.name: [] # filter on site names - website.name: ["test.local"] + # website.name: [] From 9645ff0eaf85fedc676d4908334cfffea3d980c2 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 14 Feb 2020 16:24:26 +0100 Subject: [PATCH 28/36] perfmon fix --- metricbeat/module/windows/perfmon/perfmon.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 70b0ec1402ff..4b2282777b22 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -43,8 +43,8 @@ type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment - FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment + GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment + FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment } const metricsetName = "perfmon" From ef247b80e792cf283025054c0b78884d13bff011 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 17 Feb 2020 13:13:52 +0100 Subject: [PATCH 29/36] work on websie --- metricbeat/docs/modules/iis.asciidoc | 6 ------ metricbeat/metricbeat.reference.yml | 2 -- metricbeat/module/iis/_meta/config.yml | 2 -- metricbeat/module/iis/_meta/docs.asciidoc | 4 ---- metricbeat/module/iis/website/_meta/docs.asciidoc | 3 --- metricbeat/module/iis/website/manifest.yml | 4 +++- metricbeat/module/windows/perfmon/perfmon.go | 5 +++-- metricbeat/module/windows/perfmon/reader.go | 9 +++++---- metricbeat/modules.d/iis.yml.disabled | 2 -- 9 files changed, 11 insertions(+), 26 deletions(-) diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index ca587f1d7e51..09e5c564f077 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -32,8 +32,6 @@ The `iis` module mericsets are `webserver`, `website` and `application_pool`. # filter on application pool names # application_pool.name: [] - # filter on site names - # website.name: [] ---- [float] @@ -55,8 +53,6 @@ This metricset will collect metrics of specific application pools, users can con [float] === Module-specific configuration notes -`website.name`:: []string, users can specify the sites they would like to monitor. - `application_pool.name`:: []string, users can specify the application pools they would like to monitor. @@ -80,8 +76,6 @@ metricbeat.modules: # filter on application pool names # application_pool.name: [] - # filter on site names - # website.name: [] ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 690571ccbe2d..72a96b434820 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -357,8 +357,6 @@ metricbeat.modules: # filter on application pool names # application_pool.name: [] - # filter on site names - # website.name: [] #------------------------------- Jolokia Module ------------------------------- - module: jolokia diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml index c467d4163462..a2ea8fc2e7d8 100644 --- a/metricbeat/module/iis/_meta/config.yml +++ b/metricbeat/module/iis/_meta/config.yml @@ -8,5 +8,3 @@ # filter on application pool names # application_pool.name: [] - # filter on site names - # website.name: [] diff --git a/metricbeat/module/iis/_meta/docs.asciidoc b/metricbeat/module/iis/_meta/docs.asciidoc index dba0b21ed538..a5cc75059136 100644 --- a/metricbeat/module/iis/_meta/docs.asciidoc +++ b/metricbeat/module/iis/_meta/docs.asciidoc @@ -23,8 +23,6 @@ The `iis` module mericsets are `webserver`, `website` and `application_pool`. # filter on application pool names # application_pool.name: [] - # filter on site names - # website.name: [] ---- [float] @@ -46,7 +44,5 @@ This metricset will collect metrics of specific application pools, users can con [float] === Module-specific configuration notes -`website.name`:: []string, users can specify the sites they would like to monitor. - `application_pool.name`:: []string, users can specify the application pools they would like to monitor. diff --git a/metricbeat/module/iis/website/_meta/docs.asciidoc b/metricbeat/module/iis/website/_meta/docs.asciidoc index 9f09d47fef9d..e4e6a5e9dc99 100644 --- a/metricbeat/module/iis/website/_meta/docs.asciidoc +++ b/metricbeat/module/iis/website/_meta/docs.asciidoc @@ -7,9 +7,6 @@ The metrics contain the IIS Performance counter values like: Web Service: Current Connections (through experience with their apps app, users can identify what is a normal value for this) and others. -Users can specify the sites they would like to monitor using the configuration option `website.name`, else, all websites are considered. - - diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index 8c92c28c636a..a8d0a3d86abf 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -5,7 +5,6 @@ input: defaults: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true - perfmon.filter_by_instance: [""] perfmon.counters: #network - instance_label: 'name' @@ -51,3 +50,6 @@ input: measurement_label: service_uptime query: '\Web Service(*)\Service Uptime' +processors: +- drop_event.when.equals: + iis.website.name: '_Total' diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 4b2282777b22..3ac46365a2a3 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -43,8 +43,9 @@ type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment - FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment + GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment + // could be used in the future if any solutions are built for https://github.com/elastic/beats/issues/16366 + //FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment } const metricsetName = "perfmon" diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index a9eb11c5f73f..5bd13f5d59e7 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -151,9 +151,10 @@ func (this *Reader) Read(metricset string) ([]mb.Event, error) { } var events []mb.Event if metricsetName != metricset { - if len(this.config.FilterByInstance) > 0 { - values = filterValuesByInstances(this.config.FilterByInstance, values) - } + // will be enabled when https://github.com/elastic/beats/issues/16366 + //if len(this.config.FilterByInstance) > 0 { + // values = filterValuesByInstances(this.config.FilterByInstance, values) + //} if this.config.GroupAllCountersTo != "" { event := this.groupToEvent(values) events = append(events, event) @@ -277,7 +278,7 @@ func filterValuesByInstances(instances []string, counterValues map[string][]pdh. for key, values := range counterValues { filteredValues[key] = []pdh.CounterValue{} for _, value := range values { - if containsInstance(value, instances) && value.Instance != "_Total" { + if containsInstance(value, instances) { filteredValues[key] = append(filteredValues[key], value) } } diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml.disabled index 85b3253451ac..f81d67eedffa 100644 --- a/metricbeat/modules.d/iis.yml.disabled +++ b/metricbeat/modules.d/iis.yml.disabled @@ -11,5 +11,3 @@ # filter on application pool names # application_pool.name: [] - # filter on site names - # website.name: [] From 2e4435773d6f55a4bbab143fdd35bc8d3786a469 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 18 Feb 2020 10:54:01 +0100 Subject: [PATCH 30/36] update config --- x-pack/metricbeat/metricbeat.reference.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 28ce31cc4005..35d24054fe86 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -487,6 +487,18 @@ metricbeat.modules: # the options for this metricset are also available here. metrics_path: /metrics +#--------------------------------- Iis Module --------------------------------- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] + #-------------------------------- Istio Module -------------------------------- # Istio mesh. To collect all Mixer-generated metrics - module: istio From ecaa80f033a5258e2ae61c5c998cc52be7266a12 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 19 Feb 2020 17:01:45 +0100 Subject: [PATCH 31/36] feedback --- metricbeat/docs/modules/iis.asciidoc | 4 +- metricbeat/helper/windows/pdh/pdh_windows.go | 7 +- metricbeat/module/iis/_meta/docs.asciidoc | 4 +- .../iis/application_pool/application_pool.go | 9 ++ .../module/iis/application_pool/reader.go | 4 +- metricbeat/module/windows/perfmon/perfmon.go | 1 + metricbeat/module/windows/perfmon/reader.go | 105 +++++++----------- 7 files changed, 60 insertions(+), 74 deletions(-) diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc index 09e5c564f077..50dae38e10fc 100644 --- a/metricbeat/docs/modules/iis.asciidoc +++ b/metricbeat/docs/modules/iis.asciidoc @@ -39,10 +39,12 @@ The `iis` module mericsets are `webserver`, `website` and `application_pool`. [float] === `webserver` -This metricset allows users to retrieve aggregated metrics for the entire webserver. +A light metricset using the windows perfmon metricset as the base metricset. +This metricset allows users to retrieve aggregated metrics for the entire webserver, [float] === `website` +A light metricset using the windows perfmon metricset as the base metricset. This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered. [float] diff --git a/metricbeat/helper/windows/pdh/pdh_windows.go b/metricbeat/helper/windows/pdh/pdh_windows.go index 36d375aabca6..3aa0ef25e41f 100644 --- a/metricbeat/helper/windows/pdh/pdh_windows.go +++ b/metricbeat/helper/windows/pdh/pdh_windows.go @@ -51,7 +51,8 @@ type PdhCounterHandle uintptr var InvalidCounterHandle = ^PdhCounterHandle(0) -const PERF_DETAIL_WIZARD = 400 +// counter detail level +const PerformanceDetailWizard = 400 // PdhCounterInfo struct contains the performance counter details type PdhCounterInfo struct { @@ -267,7 +268,7 @@ func PdhEnumObjectItems(objectName string) ([]uint16, []uint16, error) { &cBuffSize, &iBuff[0], &iBuffSize, - PERF_DETAIL_WIZARD, + PerformanceDetailWizard, 0); err != nil { if PdhErrno(err.(syscall.Errno)) != PDH_MORE_DATA { return nil, nil, PdhErrno(err.(syscall.Errno)) @@ -283,7 +284,7 @@ func PdhEnumObjectItems(objectName string) ([]uint16, []uint16, error) { &cBuffSize, &iBuff[0], &iBuffSize, - PERF_DETAIL_WIZARD, + PerformanceDetailWizard, 0); err != nil { return nil, nil, err } diff --git a/metricbeat/module/iis/_meta/docs.asciidoc b/metricbeat/module/iis/_meta/docs.asciidoc index a5cc75059136..309b37aeee77 100644 --- a/metricbeat/module/iis/_meta/docs.asciidoc +++ b/metricbeat/module/iis/_meta/docs.asciidoc @@ -30,10 +30,12 @@ The `iis` module mericsets are `webserver`, `website` and `application_pool`. [float] === `webserver` -This metricset allows users to retrieve aggregated metrics for the entire webserver. +A light metricset using the windows perfmon metricset as the base metricset. +This metricset allows users to retrieve aggregated metrics for the entire webserver, [float] === `website` +A light metricset using the windows perfmon metricset as the base metricset. This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered. [float] diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index 835f05c15564..c4b2c473d011 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -96,3 +96,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } return nil } + +// Close will be called when metricbeat is stopped, should close the query. +func (m *MetricSet) Close() error { + err := m.reader.close() + if err != nil { + return errors.Wrap(err, "failed to close pdh query") + } + return nil +} diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index c0f2970cb7ae..fabd3996c68b 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -215,7 +215,7 @@ func getApplicationPools(names []string) ([]ApplicationPool, error) { return filtered, nil } -// getw3wpProceses func retrieves the running w3wp process ids +// getw3wpProceses func retrieves the running w3wp process ids. A worker process is a windows process (w3wp.exe) which runs Web applications, and is responsible for handling requests sent to a Web Server for a specific application pool. func getw3wpProceses() (map[int]string, error) { processes, err := sysinfo.Processes() if err != nil { @@ -232,7 +232,7 @@ func getw3wpProceses() (map[int]string, error) { for i, ar := range info.Args { if ar == "-ap" && len(info.Args) > i+1 { wps[info.PID] = info.Args[i+1] - continue + break } } } diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 3ac46365a2a3..a9401b2bef0f 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -139,6 +139,7 @@ func (m *MetricSet) Close() error { return nil } +// validateCounterConfig func will check if the instanceLabel has been set, the instance label is mandatory inside the pefrmon metricset func validateCounterConfig(counters []CounterConfig) error { for _, counter := range counters { if counter.InstanceLabel == "" { diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 5bd13f5d59e7..0a41e7d13cfa 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -99,16 +99,16 @@ func NewReader(config Config) (*Reader, error) { } // RefreshCounterPaths will recheck for any new instances and add them to the counter list -func (this *Reader) RefreshCounterPaths() error { +func (re *Reader) RefreshCounterPaths() error { var newCounters []string - for _, counter := range this.config.CounterConfig { - childQueries, err := this.query.GetCounterPaths(counter.Query) + for _, counter := range re.config.CounterConfig { + childQueries, err := re.query.GetCounterPaths(counter.Query) if err != nil { - if this.config.IgnoreNECounters { + if re.config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: - this.log.Infow("Ignoring non existent counter", "error", err, + re.log.Infow("Ignoring non existent counter", "error", err, logp.Namespace("perfmon"), "query", counter.Query) continue } @@ -120,15 +120,15 @@ func (this *Reader) RefreshCounterPaths() error { // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { - if err := this.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { + if err := re.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } - this.instanceLabel[v] = counter.InstanceLabel - this.measurement[v] = counter.MeasurementLabel + re.instanceLabel[v] = counter.InstanceLabel + re.measurement[v] = counter.MeasurementLabel } } } - err := this.query.RemoveUnusedCounters(newCounters) + err := re.query.RemoveUnusedCounters(newCounters) if err != nil { return errors.Wrap(err, "failed removing unused counter values") } @@ -137,50 +137,45 @@ func (this *Reader) RefreshCounterPaths() error { } // Read executes a query and returns those values in an event. -func (this *Reader) Read(metricset string) ([]mb.Event, error) { +func (re *Reader) Read(metricset string) ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if err := this.query.CollectData(); err != nil { + if err := re.query.CollectData(); err != nil { return nil, errors.Wrap(err, "failed querying counter values") } // Get the values. - values, err := this.query.GetFormattedCounterValues() + values, err := re.query.GetFormattedCounterValues() if err != nil { return nil, errors.Wrap(err, "failed formatting counter values") } var events []mb.Event - if metricsetName != metricset { - // will be enabled when https://github.com/elastic/beats/issues/16366 - //if len(this.config.FilterByInstance) > 0 { - // values = filterValuesByInstances(this.config.FilterByInstance, values) - //} - if this.config.GroupAllCountersTo != "" { - event := this.groupToEvent(values) - events = append(events, event) - } + // GroupAllCountersTo config option will only apply to the webserver metricset, where counters for all instances are aggregated + if metricsetName != metricset && re.config.GroupAllCountersTo != "" { + event := re.groupToEvent(values) + events = append(events, event) } else { - events = this.groupToEvents(values) + events = re.groupToEvents(values) } - this.executed = true + re.executed = true return events, nil } -func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { +func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { eventMap := make(map[string]*mb.Event) for counterPath, values := range counters { for ind, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !this.executed { - this.log.Debugw("Ignoring the first measurement because the data isn't ready", + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } var eventKey string - if this.config.GroupMeasurements && val.Err == nil { + if re.config.GroupMeasurements && val.Err == nil { // Send measurements with the same instance label as part of the same event eventKey = val.Instance } else { @@ -195,20 +190,20 @@ func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.E MetricSetFields: common.MapStr{}, Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), } - if val.Instance != "" && this.instanceLabel[counterPath] != "" { + if val.Instance != "" && re.instanceLabel[counterPath] != "" { //will ignore instance counter if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], match) + eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], match) } else { - eventMap[eventKey].MetricSetFields.Put(this.instanceLabel[counterPath], val.Instance) + eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], val.Instance) } } } event := eventMap[eventKey] if val.Measurement != nil { - event.MetricSetFields.Put(this.measurement[counterPath], val.Measurement) + event.MetricSetFields.Put(re.measurement[counterPath], val.Measurement) } else { - event.MetricSetFields.Put(this.measurement[counterPath], 0) + event.MetricSetFields.Put(re.measurement[counterPath], 0) } } } @@ -220,7 +215,7 @@ func (this *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.E return events } -func (this *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { +func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { event := mb.Event{ MetricSetFields: common.MapStr{}, } @@ -229,18 +224,18 @@ func (this *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Even for _, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !this.executed { - this.log.Debugw("Ignoring the first measurement because the data isn't ready", + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } - if _, ok := measurements[this.measurement[counterPath]]; !ok { - measurements[this.measurement[counterPath]] = val.Measurement.(float64) - measurements[this.measurement[counterPath]+instanceCountLabel] = 1 + if _, ok := measurements[re.measurement[counterPath]]; !ok { + measurements[re.measurement[counterPath]] = val.Measurement.(float64) + measurements[re.measurement[counterPath]+instanceCountLabel] = 1 } else { - measurements[this.measurement[counterPath]+instanceCountLabel] = measurements[this.measurement[counterPath]+instanceCountLabel] + 1 - measurements[this.measurement[counterPath]] = measurements[this.measurement[counterPath]] + val.Measurement.(float64) + measurements[re.measurement[counterPath]+instanceCountLabel] = measurements[re.measurement[counterPath]+instanceCountLabel] + 1 + measurements[re.measurement[counterPath]] = measurements[re.measurement[counterPath]] + val.Measurement.(float64) } } } @@ -249,7 +244,7 @@ func (this *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Even if val == 1 { continue } else { - event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], this.config.GroupAllCountersTo), val) + event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], re.config.GroupAllCountersTo), val) } } else { event.MetricSetFields.Put(key, val) @@ -259,8 +254,8 @@ func (this *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Even } // Close will close the PDH query for now. -func (this *Reader) Close() error { - return this.query.Close() +func (re *Reader) Close() error { + return re.query.Close() } // matchParentProcess will try to get the parent process name @@ -271,27 +266,3 @@ func matchesParentProcess(instanceName string) (bool, string) { } return false, instanceName } - -// filterValuesByInstances func filters the counter valus based on the instances provided by the user, this is applied to the 'website' metricset at the moment so the _Total instance should be filtered out -func filterValuesByInstances(instances []string, counterValues map[string][]pdh.CounterValue) map[string][]pdh.CounterValue { - filteredValues := make(map[string][]pdh.CounterValue) - for key, values := range counterValues { - filteredValues[key] = []pdh.CounterValue{} - for _, value := range values { - if containsInstance(value, instances) { - filteredValues[key] = append(filteredValues[key], value) - } - } - } - return filteredValues -} - -// containsInstance func checks if the counter value contains a filtered instance -func containsInstance(counterValue pdh.CounterValue, array []string) bool { - for _, ins := range array { - if ins == counterValue.Instance { - return true - } - } - return false -} From fcfd4f398d0dad46b14fb6f3bfda66a14ec315c2 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 21 Feb 2020 12:57:38 +0100 Subject: [PATCH 32/36] work on feedback --- metricbeat/docs/fields.asciidoc | 10 ------- metricbeat/helper/windows/pdh/pdh_windows.go | 2 +- .../iis/application_pool/_meta/data.json | 2 +- .../iis/application_pool/_meta/fields.yml | 4 --- .../module/iis/application_pool/reader.go | 6 +++-- metricbeat/module/iis/fields.go | 2 +- .../windows/perfmon/_meta/docs.asciidoc | 3 +-- metricbeat/module/windows/perfmon/perfmon.go | 26 ++++--------------- metricbeat/module/windows/perfmon/reader.go | 6 ++--- .../module/windows/perfmon/reader_test.go | 7 ++--- 10 files changed, 20 insertions(+), 48 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6ba7faaf95d9..b1113adcca0a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16951,16 +16951,6 @@ application_pool application pool name -type: keyword - --- - -*`iis.application_pool.worker_processor_id`*:: -+ --- -worker processor id - - type: keyword -- diff --git a/metricbeat/helper/windows/pdh/pdh_windows.go b/metricbeat/helper/windows/pdh/pdh_windows.go index 3aa0ef25e41f..ccfb2ea50f38 100644 --- a/metricbeat/helper/windows/pdh/pdh_windows.go +++ b/metricbeat/helper/windows/pdh/pdh_windows.go @@ -51,7 +51,7 @@ type PdhCounterHandle uintptr var InvalidCounterHandle = ^PdhCounterHandle(0) -// counter detail level +// PerformanceDetailWizard is the counter detail level const PerformanceDetailWizard = 400 // PdhCounterInfo struct contains the performance counter details diff --git a/metricbeat/module/iis/application_pool/_meta/data.json b/metricbeat/module/iis/application_pool/_meta/data.json index 60e93f1ce0c4..25430c1898b0 100644 --- a/metricbeat/module/iis/application_pool/_meta/data.json +++ b/metricbeat/module/iis/application_pool/_meta/data.json @@ -12,9 +12,9 @@ "net_clr" : { "total_exceptions_thrown" : 0 }, - "worker_process_id" : 11956, "name" : "test.local", "process" : { + "pid" : 11956, "thread_count" : 32, "private_byte" : 3.4684928E7, "handle_count" : 743, diff --git a/metricbeat/module/iis/application_pool/_meta/fields.yml b/metricbeat/module/iis/application_pool/_meta/fields.yml index 10a5e4d0632c..48b70c26101e 100644 --- a/metricbeat/module/iis/application_pool/_meta/fields.yml +++ b/metricbeat/module/iis/application_pool/_meta/fields.yml @@ -8,7 +8,3 @@ type: keyword description: > application pool name - - name: worker_processor_id - type: keyword - description: > - worker processor id diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index fabd3996c68b..6e7dabd7937c 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -55,7 +55,7 @@ type WorkerProcess struct { } var appPoolCounters = map[string]string{ - "worker_process_id": "\\Process(w3wp*)\\ID Process", + "process.pid": "\\Process(w3wp*)\\ID Process", "process.cpu_usage_perc": "\\Process(w3wp*)\\% Processor Time", "process.handle_count": "\\Process(w3wp*)\\Handle Count", "process.thread_count": "\\Process(w3wp*)\\Thread Count", @@ -215,7 +215,9 @@ func getApplicationPools(names []string) ([]ApplicationPool, error) { return filtered, nil } -// getw3wpProceses func retrieves the running w3wp process ids. A worker process is a windows process (w3wp.exe) which runs Web applications, and is responsible for handling requests sent to a Web Server for a specific application pool. +// getw3wpProceses func retrieves the running w3wp process ids. +// A worker process is a windows process (w3wp.exe) which runs Web applications, +// and is responsible for handling requests sent to a Web Server for a specific application pool. func getw3wpProceses() (map[int]string, error) { processes, err := sysinfo.Processes() if err != nil { diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go index 07a039e383dc..20cec62b75e3 100644 --- a/metricbeat/module/iis/fields.go +++ b/metricbeat/module/iis/fields.go @@ -32,5 +32,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/iis. func AssetIis() string { - return "eJy0kktuwzAMRPc6xcDLAMkBtOhVDMVmAjayKUhKA9++UGW7/iNAUS68IAczzyLPeFCnwRwUEDla0iiYQ6GAmkLl2UWWVuNDAUg6NFI/LSnAkyUTSONK0SgF3JhsHfSP8ozWNDQ4p4qdI427l6frOxsBc5OpkXHOcmWSuHQidhRsOaea4/32N1NzHWQssaZo6TsbDEgP6l7i68XsAGABgQSxth9yX+If5EvnpaIQxJe8jPoDRjbHaI6J+QhA10D+i/zldDm98fQZR66fVMVJOzfKPL1ZMTvDsjHOcXvvlcWpeG+tI+bY3fwTjvQfR7W2Xt/S7iXtL/BwfX1m9vsOAAD//5CFBRA=" + return "eJy0kbGOgzAQRHt/xYgSKfkAF/cryMAm2ovBlu1cxN+ffAYOiEE02YJidzTzzFzwoEGC2QsgcNAkUTD7QgAt+caxDWx6iS8BIOrQmfapSQCONClPEjUFJQRwY9Ktl3/KC3rV0eQcJwyWJO7OPO24yQSsTZZGylrNjYriyhqjZ0HOOc4a73+fTU1zkLHFWqLF7+owIT1oeBnXbm4HABsIRIi1/ZT5otqT+yF3La/liVcnIFN/UxMW67So0vWmjdo5Vp2ylvv7qCzK4twfnTHnbfYlHOgTfb5bv9e4W+J+hYcFjpnJ7zcAAP//navhqA==" } diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index 61d03b5d5696..4c90de92fdd8 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -57,8 +57,7 @@ values as shown below. "%[measurement_label]": , ---- -*`instance_label`*:: The label used to identify the counter instance. This -field is required. +*`instance_label`*:: The label used to identify the counter instance. *`instance_name`*:: The instance name to use in the event when the counter's path (`query`) does not include an instance or when you want to override the diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index a9401b2bef0f..fffd5d6cfb59 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -22,6 +22,8 @@ package perfmon import ( "strings" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -43,15 +45,13 @@ type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` // only available for the iis/webserver metricset at the moment - // could be used in the future if any solutions are built for https://github.com/elastic/beats/issues/16366 - //FilterByInstance []string `config:"perfmon.filter_by_instance"` // only available for the iis/webserver metricset at the moment + GroupAllCountersTo string `config:"perfmon.group_all_counter"` } const metricsetName = "perfmon" func init() { - mb.Registry.MustAddMetricSet("windows", metricsetName, New) + mb.Registry.MustAddMetricSet("windows", metricsetName, New, mb.WithHostParser(parse.EmptyHostParser)) } type MetricSet struct { @@ -68,12 +68,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - if metricsetName == base.Name() { - err := validateCounterConfig(config.CounterConfig) - if err != nil { - return nil, err - } - } for _, value := range config.CounterConfig { form := strings.ToLower(value.Format) switch form { @@ -115,7 +109,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "failed retrieving counters") } } - events, err := m.reader.Read(m.BaseMetricSet.Name()) + events, err := m.reader.Read() if err != nil { return errors.Wrap(err, "failed reading counters") } @@ -138,13 +132,3 @@ func (m *MetricSet) Close() error { } return nil } - -// validateCounterConfig func will check if the instanceLabel has been set, the instance label is mandatory inside the pefrmon metricset -func validateCounterConfig(counters []CounterConfig) error { - for _, counter := range counters { - if counter.InstanceLabel == "" { - return errors.Errorf("no instance label has been configured for query %s", counter.Query) - } - } - return nil -} diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 0a41e7d13cfa..58779b4ee7cd 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -137,7 +137,7 @@ func (re *Reader) RefreshCounterPaths() error { } // Read executes a query and returns those values in an event. -func (re *Reader) Read(metricset string) ([]mb.Event, error) { +func (re *Reader) Read() ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). if err := re.query.CollectData(); err != nil { @@ -150,8 +150,8 @@ func (re *Reader) Read(metricset string) ([]mb.Event, error) { return nil, errors.Wrap(err, "failed formatting counter values") } var events []mb.Event - // GroupAllCountersTo config option will only apply to the webserver metricset, where counters for all instances are aggregated - if metricsetName != metricset && re.config.GroupAllCountersTo != "" { + // GroupAllCountersTo config option where counters for all instances are aggregated and instance count is added in the event under the string value provided by this option. + if re.config.GroupAllCountersTo != "" { event := re.groupToEvent(values) events = append(events, event) } else { diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index ebcd66cc35b3..7f925d21a623 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -50,13 +50,14 @@ func TestNewReaderWithValidQueryPath(t *testing.T) { CounterConfig: []CounterConfig{counter}, } reader, err := NewReader(config) + defer reader.Close() assert.Nil(t, err) assert.NotNil(t, reader) assert.NotNil(t, reader.query) assert.NotNil(t, reader.query.Handle) assert.NotNil(t, reader.query.Counters) assert.NotZero(t, len(reader.query.Counters)) - defer reader.Close() + } // TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. @@ -73,11 +74,11 @@ func TestReadSuccessfully(t *testing.T) { } //Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we call reader.Read() twice. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - events, err := reader.Read("perfmon") + events, err := reader.Read() assert.Nil(t, err) assert.NotNil(t, events) assert.Zero(t, len(events)) - events, err = reader.Read("perfmon") + events, err = reader.Read() assert.Nil(t, err) assert.NotNil(t, events) assert.NotZero(t, len(events)) From 118f55c81891d3c422ddcb7a6ab7bafaed1ac687 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 24 Feb 2020 13:02:17 +0100 Subject: [PATCH 33/36] temp --- .../module/iis/application_pool/application_pool.go | 2 +- metricbeat/module/iis/application_pool/reader.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index c4b2c473d011..18fe4cb764ee 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -32,7 +32,7 @@ import ( // the MetricSet for each host defined in the module's configuration. After the // MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("iis", "application_pool", New) + mb.Registry.MustAddMetricSet("iis", "application_pool", New, mb.DefaultMetricSet()) } // MetricSet holds any configuration or state information. It must implement diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index 6e7dabd7937c..32d0bcc4a641 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -54,6 +54,8 @@ type WorkerProcess struct { InstanceName string } +const ecsProcessId = "process.pid" + var appPoolCounters = map[string]string{ "process.pid": "\\Process(w3wp*)\\ID Process", "process.cpu_usage_perc": "\\Process(w3wp*)\\% Processor Time", @@ -167,7 +169,11 @@ func (re *Reader) fetch(names []string) ([]mb.Event, error) { if val.Instance == appPool.Name { events[appPool.Name].MetricSetFields.Put(appPool.counters[counterPath], val.Measurement) } else if hasWorkerProcess(val.Instance, workers, appPool.WorkerProcessIds) { - events[appPool.Name].MetricSetFields.Put(re.WorkerProcesses[counterPath], val.Measurement) + if re.WorkerProcesses[counterPath] == ecsProcessId { + events[appPool.Name].RootFields.Put(re.WorkerProcesses[counterPath], val.Measurement) + } else { + events[appPool.Name].MetricSetFields.Put(re.WorkerProcesses[counterPath], val.Measurement) + } } } From f8ab97450347bb3698d1caffbe644e7fb4dda8fb Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 24 Feb 2020 13:25:28 +0100 Subject: [PATCH 34/36] ecs --- .../module/iis/application_pool/_meta/data.json | 4 +++- .../iis/application_pool/application_pool.go | 14 +++++++------- metricbeat/module/iis/application_pool/reader.go | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/metricbeat/module/iis/application_pool/_meta/data.json b/metricbeat/module/iis/application_pool/_meta/data.json index 25430c1898b0..b1c084bfa2ee 100644 --- a/metricbeat/module/iis/application_pool/_meta/data.json +++ b/metricbeat/module/iis/application_pool/_meta/data.json @@ -7,6 +7,9 @@ "service" : { "type" : "iis" }, + "process" : { + "pid" : 7664 + }, "iis" : { "application_pool" : { "net_clr" : { @@ -14,7 +17,6 @@ }, "name" : "test.local", "process" : { - "pid" : 11956, "thread_count" : 32, "private_byte" : 3.4684928E7, "handle_count" : 743, diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go index 18fe4cb764ee..913ac03eea1f 100644 --- a/metricbeat/module/iis/application_pool/application_pool.go +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -32,7 +32,7 @@ import ( // the MetricSet for each host defined in the module's configuration. After the // MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("iis", "application_pool", New, mb.DefaultMetricSet()) + mb.Registry.MustAddMetricSet("iis", "application_pool", New) } // MetricSet holds any configuration or state information. It must implement @@ -63,15 +63,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } - - if err := reader.initCounters(config.Names); err != nil { - return nil, err - } - return &MetricSet{ + ms := &MetricSet{ BaseMetricSet: base, log: logp.NewLogger("application pool"), reader: reader, - }, nil + } + if err := ms.reader.initCounters(config.Names); err != nil { + return ms, err + } + return ms, nil } // Fetch methods implements the data gathering and data conversion to the right diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index 32d0bcc4a641..001e25c5bfc9 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -156,6 +156,7 @@ func (re *Reader) fetch(names []string) ([]mb.Event, error) { MetricSetFields: common.MapStr{ "name": appPool.Name, }, + RootFields: common.MapStr{}, } for counterPath, value := range values { for _, val := range value { From fbb7cd51d959c5c1b3667352cffb8fc0d46ef2d8 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 25 Feb 2020 16:22:48 +0100 Subject: [PATCH 35/36] temp --- .../module/iis/application_pool/reader.go | 5 ++ metricbeat/module/iis/webserver/manifest.yml | 50 +++++++++++++++++++ metricbeat/module/iis/website/manifest.yml | 37 ++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index 001e25c5bfc9..b34b4af25509 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -72,6 +72,11 @@ var appPoolCounters = map[string]string{ "net_clr.filters_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Filters / sec", "net_clr.finallys_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Finallys / sec", "net_clr.throw_to_catch_depth_per_sec": "\\.NET CLR Exceptions(w3wp*)\\Throw To Catch Depth / sec", + "net_clr.GC_time_perc": "\\.NET CLR Memory(w3wp)\\% Time in GC", + "net_clr.bytes_all_heaps": "\\.NET CLR Memory(w3wp)\\# Bytes in all Heaps", + "net_clr.GC_total_commited_bytes": "\\.NET CLR Memory(w3wp)\\# Total Committed Bytes", + "net_clr.process_id": "\\.NET CLR Memory(w3wp)\\Process ID", + "net_clr.last_time_RT_checks_perc": "\\.NET CLR Security(w3wp)\\% Time in RT checks", } // newReader creates a new instance of Reader. diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 841d4f71fc34..13bfadad9551 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -50,6 +50,56 @@ input: - instance_label: '' measurement_label: network.service_uptime query: '\Web Service(_Total)\Service Uptime' + + #added + - instance_label: '' + measurement_label: network.current_anonymous_users + query: 'Web Service(*)Current Anonymous Users' + - instance_label: '' + measurement_label: network.current_nonanonymous_users + query: 'Web Service(*)Current NonAnonymous Users' + - instance_label: '' + measurement_label: network.total_anonymous_users + query: 'Web Service(*)Total Anonymous Users' + - instance_label: '' + measurement_label: network.anonymous_users_per_sec + query: 'Web Service(*)Anonymous Users/sec' + - instance_label: '' + measurement_label: network.total_nonanonymous_users + query: 'Web Service(*)Total NonAnonymous Users' + - instance_label: '' + measurement_label: network.total_not_found_errors + query: 'Web Service(*)Total Not Found Errors' + - instance_label: '' + measurement_label: network.not_found_errors_per_sec + query: 'Web Service(*)Not Found Errors/sec' + - instance_label: '' + measurement_label: network.total_locked_errors + query: 'Web Service(*)Total Locked Errors' + - instance_label: '' + measurement_label: network.locked_errors_per_sec + query: 'Web Service(*)Locked Errors/sec' + - instance_label: '' + measurement_label: network.total_CGI_requests + query: 'Web Service(*)Total CGI Requests' + + + + + + + Web Service(*)CGI Requests/sec + Web Service(*)Total ISAPI Extension Requests + Web Service(*)ISAPI Extension Requests/sec + Web Service(*)Total Files Sent + Web Service(*)Files Sent/sec + Web Service(*)Total Files Received + Web Service(*)Files Received/sec + Web Service(*)Total Files Transferred + Web Service(*)Files/sec + + + #cache - instance_label: '' measurement_label: cache.current_files_cached diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index a8d0a3d86abf..06316010f1f7 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -49,6 +49,43 @@ input: - instance_label: 'name' measurement_label: service_uptime query: '\Web Service(*)\Service Uptime' + - instance_label: 'name' + measurement_label: total_put_requests + query: '\Web Service(*)\Total PUT Requests' + - instance_label: 'name' + measurement_label: put_requests_per_sec + query: '\Web Service(*)\PUT Requests/sec' + - instance_label: 'name' + measurement_label: total_head_requests + query: '\Web Service(*)\Total Head Requests' + - instance_label: 'name' + measurement_label: head_requests_per_sec + query: '\Web Service(*)\Head Requests/sec' + - instance_label: 'name' + measurement_label: options_requests_per_sec + query: '\Web Service(*)\Options Requests/sec' + - instance_label: 'name' + measurement_label: total_options_requests + query: '\Web Service(*)\Total Options Requests' + - instance_label: 'name' + measurement_label: other_request_methods_per_sec + query: '\Web Service(*)\Other Request Methods/sec' + - instance_label: 'name' + measurement_label: total_other_request_methods + query: '\Web Service(*)\Total Other Request Methods' + - instance_label: 'name' + measurement_label: total_trace_requests + query: '\Web Service(*)\Total Trace Requests' + - instance_label: 'name' + measurement_label: trace_requests_per_sec + query: '\Web Service(*)\Trace Requests/sec' + - instance_label: 'name' + measurement_label: total_unlock_requests + query: '\Web Service(*)\Total Unlock Requests' + - instance_label: 'name' + measurement_label: unlock_requests_per_sec + query: '\Web Service(*)\Unlock Requests/sec' + processors: - drop_event.when.equals: From 5f1b016c1670e52609bc6bd5f1719424efbed5f1 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 26 Feb 2020 19:46:18 +0100 Subject: [PATCH 36/36] add counters --- .../module/iis/application_pool/reader.go | 8 +-- metricbeat/module/iis/webserver/manifest.yml | 60 ++++++++----------- metricbeat/module/iis/website/manifest.yml | 33 +--------- metricbeat/module/windows/perfmon/reader.go | 12 +++- 4 files changed, 37 insertions(+), 76 deletions(-) diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go index b34b4af25509..11651d5e510b 100644 --- a/metricbeat/module/iis/application_pool/reader.go +++ b/metricbeat/module/iis/application_pool/reader.go @@ -72,11 +72,6 @@ var appPoolCounters = map[string]string{ "net_clr.filters_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Filters / sec", "net_clr.finallys_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Finallys / sec", "net_clr.throw_to_catch_depth_per_sec": "\\.NET CLR Exceptions(w3wp*)\\Throw To Catch Depth / sec", - "net_clr.GC_time_perc": "\\.NET CLR Memory(w3wp)\\% Time in GC", - "net_clr.bytes_all_heaps": "\\.NET CLR Memory(w3wp)\\# Bytes in all Heaps", - "net_clr.GC_total_commited_bytes": "\\.NET CLR Memory(w3wp)\\# Total Committed Bytes", - "net_clr.process_id": "\\.NET CLR Memory(w3wp)\\Process ID", - "net_clr.last_time_RT_checks_perc": "\\.NET CLR Security(w3wp)\\% Time in RT checks", } // newReader creates a new instance of Reader. @@ -109,7 +104,8 @@ func (re *Reader) initCounters(filtered []string) error { for key, value := range appPoolCounters { counters, err := re.Query.ExpandWildCardPath(value) if err != nil { - return errors.Wrapf(err, `failed to expand counter path (query="%v")`, value) + re.log.Error(err, `failed to expand counter path (query="%v")`, value) + continue } for _, count := range counters { if err = re.Query.AddCounter(count, "", "float", true); err != nil { diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml index 13bfadad9551..c8d148a21c13 100644 --- a/metricbeat/module/iis/webserver/manifest.yml +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -22,7 +22,7 @@ input: query: '\Web Service(_Total)\Bytes Received/sec' - instance_label: '' measurement_label: network.current_connections - query: 'Web Service(_Total)\Current Connections' + query: '\Web Service(_Total)\Current Connections' - instance_label: '' measurement_label: network.maximum_connections query: '\Web Service(_Total)\Maximum Connections' @@ -50,55 +50,45 @@ input: - instance_label: '' measurement_label: network.service_uptime query: '\Web Service(_Total)\Service Uptime' - - #added - instance_label: '' measurement_label: network.current_anonymous_users - query: 'Web Service(*)Current Anonymous Users' + query: '\Web Service(_Total)\Current Anonymous Users' - instance_label: '' measurement_label: network.current_nonanonymous_users - query: 'Web Service(*)Current NonAnonymous Users' + query: '\Web Service(_Total)\Current NonAnonymous Users' - instance_label: '' measurement_label: network.total_anonymous_users - query: 'Web Service(*)Total Anonymous Users' + query: '\Web Service(_Total)\Total Anonymous Users' - instance_label: '' measurement_label: network.anonymous_users_per_sec - query: 'Web Service(*)Anonymous Users/sec' + query: '\Web Service(_Total)\Anonymous Users/sec' - instance_label: '' measurement_label: network.total_nonanonymous_users - query: 'Web Service(*)Total NonAnonymous Users' + query: '\Web Service(_Total)\Total NonAnonymous Users' + + #asp.net - instance_label: '' - measurement_label: network.total_not_found_errors - query: 'Web Service(*)Total Not Found Errors' + measurement_label: asp_net_application.errors_per_sec + query: '\ASP.NET Applications(__Total__)\Errors Total/Sec' - instance_label: '' - measurement_label: network.not_found_errors_per_sec - query: 'Web Service(*)Not Found Errors/sec' + measurement_label: asp_net_application.pipeline_instance_count + query: '\ASP.NET Applications(__Total__)\Pipeline Instance Count' - instance_label: '' - measurement_label: network.total_locked_errors - query: 'Web Service(*)Total Locked Errors' + measurement_label: asp_net_application.requests_executing + query: '\ASP.NET Applications(__Total__)\Requests Executing' - instance_label: '' - measurement_label: network.locked_errors_per_sec - query: 'Web Service(*)Locked Errors/sec' + measurement_label: asp_net_application.requests_in_application_queue + query: '\ASP.NET Applications(__Total__)\Requests in Application Queue' + format: 'large' - instance_label: '' - measurement_label: network.total_CGI_requests - query: 'Web Service(*)Total CGI Requests' - - - - - - - Web Service(*)CGI Requests/sec - Web Service(*)Total ISAPI Extension Requests - Web Service(*)ISAPI Extension Requests/sec - Web Service(*)Total Files Sent - Web Service(*)Files Sent/sec - Web Service(*)Total Files Received - Web Service(*)Files Received/sec - Web Service(*)Total Files Transferred - Web Service(*)Files/sec - - + measurement_label: asp_net_application.requests_per_sec + query: '\ASP.NET Applications(__Total__)\Requests/Sec' + - instance_label: '' + measurement_label: asp_net.application_restarts + query: '\ASP.NET\Application Restarts' + - instance_label: '' + measurement_label: asp_net.request_wait_time + query: '\ASP.NET\Request Wait Time' #cache - instance_label: '' diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml index 06316010f1f7..564ad6db86ce 100644 --- a/metricbeat/module/iis/website/manifest.yml +++ b/metricbeat/module/iis/website/manifest.yml @@ -21,7 +21,7 @@ input: query: '\Web Service(*)\Bytes Received/sec' - instance_label: 'name' measurement_label: current_connections - query: 'Web Service(*)\Current Connections' + query: '\Web Service(*)\Current Connections' - instance_label: 'name' measurement_label: maximum_connections query: '\Web Service(*)\Maximum Connections' @@ -55,37 +55,6 @@ input: - instance_label: 'name' measurement_label: put_requests_per_sec query: '\Web Service(*)\PUT Requests/sec' - - instance_label: 'name' - measurement_label: total_head_requests - query: '\Web Service(*)\Total Head Requests' - - instance_label: 'name' - measurement_label: head_requests_per_sec - query: '\Web Service(*)\Head Requests/sec' - - instance_label: 'name' - measurement_label: options_requests_per_sec - query: '\Web Service(*)\Options Requests/sec' - - instance_label: 'name' - measurement_label: total_options_requests - query: '\Web Service(*)\Total Options Requests' - - instance_label: 'name' - measurement_label: other_request_methods_per_sec - query: '\Web Service(*)\Other Request Methods/sec' - - instance_label: 'name' - measurement_label: total_other_request_methods - query: '\Web Service(*)\Total Other Request Methods' - - instance_label: 'name' - measurement_label: total_trace_requests - query: '\Web Service(*)\Total Trace Requests' - - instance_label: 'name' - measurement_label: trace_requests_per_sec - query: '\Web Service(*)\Trace Requests/sec' - - instance_label: 'name' - measurement_label: total_unlock_requests - query: '\Web Service(*)\Total Unlock Requests' - - instance_label: 'name' - measurement_label: unlock_requests_per_sec - query: '\Web Service(*)\Unlock Requests/sec' - processors: - drop_event.when.equals: diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 58779b4ee7cd..22b6b5f89ddd 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -229,13 +229,19 @@ func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } - + var counterVal float64 + switch val.Measurement.(type) { + case int64: + counterVal = float64(val.Measurement.(int64)) + default: + counterVal = val.Measurement.(float64) + } if _, ok := measurements[re.measurement[counterPath]]; !ok { - measurements[re.measurement[counterPath]] = val.Measurement.(float64) + measurements[re.measurement[counterPath]] = counterVal measurements[re.measurement[counterPath]+instanceCountLabel] = 1 } else { measurements[re.measurement[counterPath]+instanceCountLabel] = measurements[re.measurement[counterPath]+instanceCountLabel] + 1 - measurements[re.measurement[counterPath]] = measurements[re.measurement[counterPath]] + val.Measurement.(float64) + measurements[re.measurement[counterPath]] = measurements[re.measurement[counterPath]] + counterVal } } }