From 3e99497ae241f9bbd0a5fab907e9848d855de65d Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 7 Feb 2019 11:20:14 +0100 Subject: [PATCH 01/14] Atomic commit --- CHANGELOG.next.asciidoc | 1 + metricbeat/docker-compose.yml | 4 + metricbeat/docs/fields.asciidoc | 140 +++++++++ metricbeat/docs/modules/consul.asciidoc | 53 ++++ metricbeat/docs/modules/consul/agent.asciidoc | 23 ++ metricbeat/docs/modules_list.asciidoc | 3 + metricbeat/include/list.go | 2 + metricbeat/metricbeat.reference.yml | 9 + metricbeat/module/consul/_meta/Dockerfile | 9 + metricbeat/module/consul/_meta/config.yml | 7 + metricbeat/module/consul/_meta/docs.asciidoc | 15 + metricbeat/module/consul/_meta/env | 2 + metricbeat/module/consul/_meta/fields.yml | 9 + .../module/consul/agent/_meta/data.json | 51 ++++ .../module/consul/agent/_meta/docs.asciidoc | 13 + .../module/consul/agent/_meta/fields.yml | 72 +++++ metricbeat/module/consul/agent/agent.go | 93 ++++++ .../consul/agent/agent_integration_test.go | 64 ++++ metricbeat/module/consul/agent/agent_test.go | 288 ++++++++++++++++++ metricbeat/module/consul/agent/data.go | 133 ++++++++ .../consul/agent/data_integration_test.go | 45 +++ .../module/consul/agent/input_format.go | 55 ++++ metricbeat/module/consul/fields.go | 36 +++ metricbeat/module/consul/testing.go | 42 +++ metricbeat/modules.d/consul.yml.disabled | 10 + metricbeat/tests/system/test_consul.py | 61 ++++ x-pack/metricbeat/metricbeat.reference.yml | 8 + 27 files changed, 1248 insertions(+) create mode 100644 metricbeat/docs/modules/consul.asciidoc create mode 100644 metricbeat/docs/modules/consul/agent.asciidoc create mode 100644 metricbeat/module/consul/_meta/Dockerfile create mode 100644 metricbeat/module/consul/_meta/config.yml create mode 100644 metricbeat/module/consul/_meta/docs.asciidoc create mode 100644 metricbeat/module/consul/_meta/env create mode 100644 metricbeat/module/consul/_meta/fields.yml create mode 100644 metricbeat/module/consul/agent/_meta/data.json create mode 100644 metricbeat/module/consul/agent/_meta/docs.asciidoc create mode 100644 metricbeat/module/consul/agent/_meta/fields.yml create mode 100644 metricbeat/module/consul/agent/agent.go create mode 100644 metricbeat/module/consul/agent/agent_integration_test.go create mode 100644 metricbeat/module/consul/agent/agent_test.go create mode 100644 metricbeat/module/consul/agent/data.go create mode 100644 metricbeat/module/consul/agent/data_integration_test.go create mode 100644 metricbeat/module/consul/agent/input_format.go create mode 100644 metricbeat/module/consul/fields.go create mode 100644 metricbeat/module/consul/testing.go create mode 100644 metricbeat/modules.d/consul.yml.disabled create mode 100644 metricbeat/tests/system/test_consul.py diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8d3a4998af95..054ece2424d3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -324,6 +324,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added 'server' Metricset to Zookeeper Metricbeat module {issue}8938[8938] {pull}10341[10341] - Release AWS module as GA. {pull}10345[10345] - Add overview dashboard to Zookeeper Metricbeat module {pull}10379[10379] +- Add Consul Metricbeat module with Agent Metricset {pull}8631[8631] *Packetbeat* diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 2ac312a65a36..a2e7f60445f2 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -14,6 +14,7 @@ services: - ./module/aerospike/_meta/env - ./module/apache/_meta/env - ./module/ceph/_meta/env + - ./module/consul/_meta/env - ./module/couchbase/_meta/env - ./module/couchdb/_meta/env - ./module/dropwizard/_meta/env @@ -58,6 +59,9 @@ services: ceph: build: ./module/ceph/_meta + consul: + build: ./module/consul/_meta + couchbase: build: ./module/couchbase/_meta diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4517a8232006..f669149dd646 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19,6 +19,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2103,6 +2104,145 @@ required: True The document type. Always set to "doc". +-- + +[[exported-fields-consul]] +== consul fields + +Consul module + + + + +[float] +== agent fields + +Agent Metricset fetches metrics information from a Consul instance running as Agent + + + + + +*`consul.agent.raft.commit_time.ms`*:: ++ +-- +type: long + +Average time in milliseconds it takes to commit a new entry to the transaction log of the leader + +-- + +*`consul.agent.raft.apply`*:: ++ +-- +type: long + +Number of logs committed since the last interval. + +-- + + +*`consul.agent.autopilot.healthy`*:: ++ +-- +type: long + +Overall health of the local server cluster. If all servers are considered healthy by Autopilot, this will be set to 1. If any are unhealthy, this will be 0. + +-- + +[float] +== runtime fields + +Runtime related metrics + + + +*`consul.agent.runtime.sys.bytes`*:: ++ +-- +type: long + +Number of bytes of memory obtained from the OS. + +-- + +*`consul.agent.runtime.malloc_count`*:: ++ +-- +type: long + +Heap objects allocated + +-- + +*`consul.agent.runtime.heap_objects`*:: ++ +-- +type: long + +Objects allocated on the heap and is a general memory pressure indicator. This may burst from time to time but should return to a steady state value. + +-- + +*`consul.agent.runtime.goroutines`*:: ++ +-- +type: long + +Running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value. + +-- + + +*`consul.agent.runtime.alloc.bytes`*:: ++ +-- +type: long + +Bytes allocated by the Consul process. + +-- + +[float] +== garbage_collector fields + +Garbage collector metrics + + +*`consul.agent.runtime.garbage_collector.runs`*:: ++ +-- +type: long + +Garbage collector total executions + +-- + +[float] +== pause fields + +Time that the garbage collector has paused the app + + + +*`consul.agent.runtime.garbage_collector.pause.current.ns`*:: ++ +-- +type: long + +Garbage collector pause time in nanoseconds + +-- + + +*`consul.agent.runtime.garbage_collector.pause.total.ns`*:: ++ +-- +type: long + +Nanoseconds consumed by stop-the-world garbage collection pauses since Consul started. + -- [[exported-fields-couchbase]] diff --git a/metricbeat/docs/modules/consul.asciidoc b/metricbeat/docs/modules/consul.asciidoc new file mode 100644 index 000000000000..fce72c1c9974 --- /dev/null +++ b/metricbeat/docs/modules/consul.asciidoc @@ -0,0 +1,53 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-consul]] +== consul module + +beta[] + +This is the https://www.consul.io[Hashicorp's Consul] Metricbeat module. It is still in beta and under active development to add new Metricsets and introduce enhancements. + +[float] +== Compatibility + +The module is being tested with https://github.com/hashicorp/docker-consul/blob/9bd2aa7ecf2414b8712e055f2374699148e8941c/0.X/Dockerfile[1.4.2] version + +[float] +== Metricsets + +The following Metricset is included: + +=== `agent` + +`agent` Metricset fetches information from a Consul Agent in 'Client' mode. It fetches information about the health of the autopilot, runtime metrics and raft data. + + +[float] +=== Example configuration + +The consul module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: consul + metricsets: + - agent + enabled: true + period: 10s + hosts: ["localhost:8500"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::consul/agent.asciidoc[] + diff --git a/metricbeat/docs/modules/consul/agent.asciidoc b/metricbeat/docs/modules/consul/agent.asciidoc new file mode 100644 index 000000000000..5416da785fc2 --- /dev/null +++ b/metricbeat/docs/modules/consul/agent.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-consul-agent]] +=== consul agent metricset + +beta[] + +include::../../../module/consul/agent/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/consul/agent/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index bd96f6c37356..d32853540383 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -19,6 +19,8 @@ This file is generated! See scripts/docs_collector.py |<> |<> |<> +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .3+| .3+| |<> |<> @@ -171,6 +173,7 @@ include::modules/aerospike.asciidoc[] include::modules/apache.asciidoc[] include::modules/aws.asciidoc[] include::modules/ceph.asciidoc[] +include::modules/consul.asciidoc[] include::modules/couchbase.asciidoc[] include::modules/couchdb.asciidoc[] include::modules/docker.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index be46965fc005..eee014afda39 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -38,6 +38,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" _ "github.com/elastic/beats/metricbeat/module/ceph/pool_disk" + _ "github.com/elastic/beats/metricbeat/module/consul" + _ "github.com/elastic/beats/metricbeat/module/consul/agent" _ "github.com/elastic/beats/metricbeat/module/couchbase" _ "github.com/elastic/beats/metricbeat/module/couchbase/bucket" _ "github.com/elastic/beats/metricbeat/module/couchbase/cluster" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index d6f0d0a79f72..3d844a437ad3 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -154,6 +154,15 @@ metricbeat.modules: hosts: ["localhost:5000"] enabled: true +#------------------------------- consul Module ------------------------------- +- module: consul + metricsets: + - agent + enabled: true + period: 10s + hosts: ["localhost:8500"] + + #------------------------------ Couchbase Module ----------------------------- - module: couchbase metricsets: ["bucket", "cluster", "node"] diff --git a/metricbeat/module/consul/_meta/Dockerfile b/metricbeat/module/consul/_meta/Dockerfile new file mode 100644 index 000000000000..982e0dca81ad --- /dev/null +++ b/metricbeat/module/consul/_meta/Dockerfile @@ -0,0 +1,9 @@ +FROM consul:1.4.2 + +ENV CONSUL_BIND_INTERFACE='eth0' + +EXPOSE 8500 + +# Use the same healthcheck as the Windows version of the image. +# https://github.com/Microsoft/mssql-docker/blob/a3020afeec9be1eb2d67645ac739438eb8f2c545/windows/mssql-server-windows/dockerfile#L31 +HEALTHCHECK --interval=1s --retries=90 CMD curl http://0.0.0.0:8500/v1/agent/metrics diff --git a/metricbeat/module/consul/_meta/config.yml b/metricbeat/module/consul/_meta/config.yml new file mode 100644 index 000000000000..0f071def28be --- /dev/null +++ b/metricbeat/module/consul/_meta/config.yml @@ -0,0 +1,7 @@ +- module: consul + metricsets: + - agent + enabled: true + period: 10s + hosts: ["localhost:8500"] + diff --git a/metricbeat/module/consul/_meta/docs.asciidoc b/metricbeat/module/consul/_meta/docs.asciidoc new file mode 100644 index 000000000000..4bd1eb510e7d --- /dev/null +++ b/metricbeat/module/consul/_meta/docs.asciidoc @@ -0,0 +1,15 @@ +This is the https://www.consul.io[Hashicorp's Consul] Metricbeat module. It is still in beta and under active development to add new Metricsets and introduce enhancements. + +[float] +== Compatibility + +The module is being tested with https://github.com/hashicorp/docker-consul/blob/9bd2aa7ecf2414b8712e055f2374699148e8941c/0.X/Dockerfile[1.4.2] version + +[float] +== Metricsets + +The following Metricset is included: + +=== `agent` + +`agent` Metricset fetches information from a Consul Agent in 'Client' mode. It fetches information about the health of the autopilot, runtime metrics and raft data. diff --git a/metricbeat/module/consul/_meta/env b/metricbeat/module/consul/_meta/env new file mode 100644 index 000000000000..211d9aa87693 --- /dev/null +++ b/metricbeat/module/consul/_meta/env @@ -0,0 +1,2 @@ +CONSUL_HOST=consul +CONSUL_PORT=8500 diff --git a/metricbeat/module/consul/_meta/fields.yml b/metricbeat/module/consul/_meta/fields.yml new file mode 100644 index 000000000000..c6a4d8956131 --- /dev/null +++ b/metricbeat/module/consul/_meta/fields.yml @@ -0,0 +1,9 @@ +- key: consul + title: "consul" + description: > + Consul module + release: beta + fields: + - name: consul + type: group + fields: diff --git a/metricbeat/module/consul/agent/_meta/data.json b/metricbeat/module/consul/agent/_meta/data.json new file mode 100644 index 000000000000..86782e37258b --- /dev/null +++ b/metricbeat/module/consul/agent/_meta/data.json @@ -0,0 +1,51 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "consul": { + "agent": { + "autopilot": { + "healthy": 1 + }, + "raft": { + "apply": 1, + "commit_time": { + "ms": 0.01523400004953146 + } + }, + "runtime": { + "alloc": { + "bytes": 8543432 + }, + "garbage_collector": { + "pause": { + "total": { + "ns": 43749010 + } + }, + "runs": 101 + }, + "goroutines": 76, + "heap_objects": 48889, + "malloc_count": 3200832, + "sys": { + "bytes": 74119416 + } + } + } + }, + "event": { + "dataset": "consul.agent", + "duration": 115000, + "module": "consul" + }, + "metricset": { + "name": "agent" + }, + "service": { + "address": "localhost:8500", + "type": "consul" + } +} diff --git a/metricbeat/module/consul/agent/_meta/docs.asciidoc b/metricbeat/module/consul/agent/_meta/docs.asciidoc new file mode 100644 index 000000000000..483ead431fc8 --- /dev/null +++ b/metricbeat/module/consul/agent/_meta/docs.asciidoc @@ -0,0 +1,13 @@ +This is the 'agent' Metricset of the Hashicorp's Consul Metricbeat module. + +* *agent.autopilot.healthy*: Tracks the overall health of the local server cluster. If all servers are considered healthy by Autopilot, this will be set to 1. If any are unhealthy, this will be 0. +* *agent.raft.apply*: This metric gives the number of logs committed since the last interval. +* *agent.raft.commit_time.ms*: This tracks the average time in milliseconds it takes to commit a new entry to the transaction log of the leader +* *agent.runtime.alloc.bytes*: This measures the number of bytes allocated by the Consul process. +* *agent.runtime.garbage_collector.pause.current.ns*: Garbage collector pause time in nanoseconds +* *agent.runtime.garbage_collector.pause.total.ns*: Number of nanoseconds consumed by stop-the-world garbage collection pauses since Consul started. +* *agent.runtime.garbage_collector.runs*: Garbage collector total executions +* *agent.runtime.goroutines*: Number of running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value. +* *agent.runtime.heap_objects*: This measures the number of objects allocated on the heap and is a general memory pressure indicator. This may burst from time to time but should return to a steady state value. +* *agent.runtime.malloc_count*: Heap objects allocated +* *agent.runtime.sys.bytes*: Total number of bytes of memory obtained from the OS. diff --git a/metricbeat/module/consul/agent/_meta/fields.yml b/metricbeat/module/consul/agent/_meta/fields.yml new file mode 100644 index 000000000000..657a52ff7c9b --- /dev/null +++ b/metricbeat/module/consul/agent/_meta/fields.yml @@ -0,0 +1,72 @@ +- name: agent + type: group + release: beta + description: > + Agent Metricset fetches metrics information from a Consul instance running as Agent + fields: + - name: raft + type: group + fields: + - name: commit_time + type: group + fields: + - name: ms + type: long + description: Average time in milliseconds it takes to commit a new entry to the transaction log of the leader + - name: apply + type: long + description: Number of logs committed since the last interval. + - name: autopilot + type: group + fields: + - name: healthy + type: long + description: Overall health of the local server cluster. If all servers are considered healthy by Autopilot, this will be set to 1. If any are unhealthy, this will be 0. + - name: runtime + type: group + description: Runtime related metrics + fields: + - name: sys + type: group + fields: + - name: bytes + type: long + description: Number of bytes of memory obtained from the OS. + - name: malloc_count + type: long + description: Heap objects allocated + - name: heap_objects + type: long + description: Objects allocated on the heap and is a general memory pressure indicator. This may burst from time to time but should return to a steady state value. + - name: goroutines + type: long + description: Running goroutines and is a general load pressure indicator. This may burst from time to time but should return to a steady state value. + - name: alloc + type: group + fields: + - name: bytes + type: long + description: Bytes allocated by the Consul process. + - name: garbage_collector + type: group + description: Garbage collector metrics + fields: + - name: runs + type: long + description: Garbage collector total executions + - name: pause + type: group + description: Time that the garbage collector has paused the app + fields: + - name: current + type: group + fields: + - name: ns + type: long + description: Garbage collector pause time in nanoseconds + - name: total + type: group + fields: + - name: ns + type: long + description: Nanoseconds consumed by stop-the-world garbage collection pauses since Consul started. diff --git a/metricbeat/module/consul/agent/agent.go b/metricbeat/module/consul/agent/agent.go new file mode 100644 index 000000000000..aa353ada4340 --- /dev/null +++ b/metricbeat/module/consul/agent/agent.go @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package agent + +import ( + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: "/v1/agent/metrics", + }.Build() +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("consul", "agent", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet()) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The consul agent metricset is beta.") + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + content, err := m.http.FetchContent() + if err != nil { + logp.Error(err) + report.Error(err) + return + } + + mappings, err := eventMapping(content) + if err != nil { + logp.Error(err) + report.Error(err) + return + } + + for _, m := range mappings { + report.Event(mb.Event{ + MetricSetFields: m, + }) + } +} diff --git a/metricbeat/module/consul/agent/agent_integration_test.go b/metricbeat/module/consul/agent/agent_integration_test.go new file mode 100644 index 000000000000..84d9181ef6f2 --- /dev/null +++ b/metricbeat/module/consul/agent/agent_integration_test.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package agent + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/tests/compose" + "github.com/elastic/beats/metricbeat/module/consul" + + "github.com/elastic/beats/libbeat/logp" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + logp.TestingSetup() + + compose.EnsureUp(t, "consul") + + f := mbtest.NewReportingMetricSetV2(t, consul.GetConfig([]string{"agent"})) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + t.Logf("Found '%d' events", len(events)) + + for _, event := range events { + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + metricsetFields := event.MetricSetFields + + // Check runtime value + runtime, ok := metricsetFields["runtime"].(common.MapStr) + assert.True(t, ok) + + //Check heapObjects + heapObjects, ok := runtime["heap_objects"].(float64) + assert.True(t, ok) + assert.True(t, heapObjects > float64(0)) + + } +} diff --git a/metricbeat/module/consul/agent/agent_test.go b/metricbeat/module/consul/agent/agent_test.go new file mode 100644 index 000000000000..9474b63e5992 --- /dev/null +++ b/metricbeat/module/consul/agent/agent_test.go @@ -0,0 +1,288 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package agent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +var jsonExample = `{ + "Timestamp": "2019-02-01 11:08:50 +0000 UTC", + "Gauges": [ + { + "Name": "consul.autopilot.failure_tolerance", + "Value": 0, + "Labels": {} + }, + { + "Name": "consul.autopilot.healthy", + "Value": 1, + "Labels": {} + }, + { + "Name": "consul.runtime.alloc_bytes", + "Value": 5034304, + "Labels": {} + }, + { + "Name": "consul.runtime.free_count", + "Value": 1202914, + "Labels": {} + }, + { + "Name": "consul.runtime.heap_objects", + "Value": 35836, + "Labels": {"service":"service1"} + }, + { + "Name": "consul.runtime.malloc_count", + "Value": 1238750, + "Labels": {} + }, + { + "Name": "consul.runtime.num_goroutines", + "Value": 76, + "Labels": {} + }, + { + "Name": "consul.runtime.sys_bytes", + "Value": 73070840, + "Labels": {} + }, + { + "Name": "consul.runtime.total_gc_pause_ns", + "Value": 7107735, + "Labels": {} + }, + { + "Name": "consul.runtime.total_gc_runs", + "Value": 42, + "Labels": {} + }, + { + "Name": "consul.session_ttl.active", + "Value": 0, + "Labels": {} + } + ], + "Points": [], + "Counters": [ + { + "Name": "consul.raft.apply", + "Count": 1, + "Rate": 0.1, + "Sum": 1, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.rpc.query", + "Count": 2, + "Rate": 0.2, + "Sum": 2, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.rpc.request", + "Count": 5, + "Rate": 0.5, + "Sum": 5, + "Min": 1, + "Max": 1, + "Mean": 1, + "Stddev": 0, + "Labels": {} + } + ], + "Samples": [ + { + "Name": "consul.fsm.coordinate.batch-update", + "Count": 1, + "Rate": 0.003936899825930595, + "Sum": 0.039368998259305954, + "Min": 0.039368998259305954, + "Max": 0.039368998259305954, + "Mean": 0.039368998259305954, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.http.GET.v1.agent.metrics", + "Count": 10, + "Rate": 0.2068565994501114, + "Sum": 2.068565994501114, + "Min": 0.14361299574375153, + "Max": 0.46759501099586487, + "Mean": 0.2068565994501114, + "Stddev": 0.09421784218829098, + "Labels": {} + }, + { + "Name": "consul.memberlist.gossip", + "Count": 200, + "Rate": 0.2729187995195389, + "Sum": 2.729187995195389, + "Min": 0.0022559999488294125, + "Max": 0.10744299739599228, + "Mean": 0.013645939975976944, + "Stddev": 0.013672823772901079, + "Labels": {} + }, + { + "Name": "consul.raft.commitTime", + "Count": 1, + "Rate": 0.00219310000538826, + "Sum": 0.0219310000538826, + "Min": 0.0219310000538826, + "Max": 0.0219310000538826, + "Mean": 0.0219310000538826, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.raft.fsm.apply", + "Count": 1, + "Rate": 0.005605699867010117, + "Sum": 0.056056998670101166, + "Min": 0.056056998670101166, + "Max": 0.056056998670101166, + "Mean": 0.056056998670101166, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.raft.leader.dispatchLog", + "Count": 1, + "Rate": 0.001807899959385395, + "Sum": 0.01807899959385395, + "Min": 0.01807899959385395, + "Max": 0.01807899959385395, + "Mean": 0.01807899959385395, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.runtime.gc_pause_ns", + "Count": 1, + "Rate": 12140.1, + "Sum": 121401, + "Min": 121401, + "Max": 121401, + "Mean": 121401, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.serf.queue.Event", + "Count": 2, + "Rate": 0.1, + "Sum": 1, + "Min": 0, + "Max": 1, + "Mean": 0.5, + "Stddev": 0.7071067811865476, + "Labels": {} + }, + { + "Name": "consul.serf.queue.Intent", + "Count": 2, + "Rate": 0, + "Sum": 0, + "Min": 0, + "Max": 0, + "Mean": 0, + "Stddev": 0, + "Labels": {} + }, + { + "Name": "consul.serf.queue.Query", + "Count": 2, + "Rate": 0, + "Sum": 0, + "Min": 0, + "Max": 0, + "Mean": 0, + "Stddev": 0, + "Labels": {} + } + ] +}` + +func TestEventMapping(t *testing.T) { + byt := []byte(jsonExample) + + events, err := eventMapping(byt) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 2, len(events)) + + //2 events should be here, one with runtime.heap_objects only and one with everything else + heapObjectsFound := false + var heapObjects interface{} + + goroutinesFound := false + var goroutines interface{} + + for _, event := range events { + runtimeI, ok := event["runtime"] + assert.True(t, ok) + + runtime, ok := runtimeI.(common.MapStr) + assert.True(t, ok) + + //do not overwrite if heapObjectsFound has already been set to true + if !heapObjectsFound { + heapObjects, heapObjectsFound = runtime["heap_objects"] + if heapObjectsFound { + heapObjectsFloat64, ok := heapObjects.(float64) + assert.True(t, ok) + + assert.True(t, heapObjectsFloat64 > 0) + } + } + + //do not overwrite if goroutinesFound has already been set to true + if !goroutinesFound { + goroutines, goroutinesFound = runtime["goroutines"] + if goroutinesFound { + goroutinesFloat64, ok := goroutines.(float64) + assert.True(t, ok) + + assert.True(t, goroutinesFloat64 > 0) + } + } + } + + assert.True(t, goroutinesFound) + assert.True(t, heapObjectsFound) +} diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go new file mode 100644 index 000000000000..d6662aba1f2d --- /dev/null +++ b/metricbeat/module/consul/agent/data.go @@ -0,0 +1,133 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package agent + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" +) + +type valueHelper struct { + Value string + Unit string +} + +var ( + allowedValues = map[string]valueHelper{ + "consul.autopilot.healthy": {Value: "autopilot.healthy"}, + "consul.runtime.alloc_bytes": {Value: "runtime.alloc", Unit: ".bytes"}, + "consul.runtime.total_gc_pause_ns": {Value: "runtime.garbage_collector.pause.total", Unit: ".ns"}, + "consul.runtime.gc_pause_ns": {Value: "runtime.garbage_collector.pause.current", Unit: ".ns"}, + "consul.runtime.total_gc_runs": {Value: "runtime.garbage_collector.runs"}, + "consul.runtime.num_goroutines": {Value: "runtime.goroutines"}, + "consul.runtime.heap_objects": {Value: "runtime.heap_objects"}, + "consul.runtime.sys_bytes": {Value: "runtime.sys", Unit: ".bytes"}, + "consul.runtime.malloc_count": {Value: "runtime.malloc_count"}, + } + allowedDetailedValues = map[string]valueHelper{ + "consul.raft.apply": {Value: "raft.apply"}, + "consul.raft.commitTime": {Value: "raft.commit_time", Unit: ".ms"}, + } +) + +func eventMapping(content []byte) ([]common.MapStr, error) { + var agent agent + + if err := json.Unmarshal(content, &agent); err != nil { + return nil, err + } + + labels := map[string]common.MapStr{} + + for _, gauge := range agent.Gauges { + metricApply(labels, gauge.consulMetric, gauge.Value) + } + + for _, point := range agent.Points { + metricApply(labels, point.consulMetric, point.Value) + } + + for _, counter := range agent.Counters { + metricApply(labels, counter.consulMetric, consulDetailedValue(counter)) + } + + for _, sample := range agent.Samples { + metricApply(labels, sample.consulMetric, consulDetailedValue(sample)) + } + + data := make([]common.MapStr, 0) + for _, v := range labels { + data = append(data, v) + } + + return data, nil +} + +func metricApply(labels map[string]common.MapStr, m consulMetric, v interface{}) { + prettyName := prettyName(m.Name) + if prettyName == nil { + //omitting unwanted metric + return + } + + labelsCombination := "" + for k, v := range m.Labels { + labelsCombination = fmt.Sprintf("%s%s%s", labelsCombination, k, v) + } + + temp := common.MapStr{} + if len(m.Labels) != 0 { + temp.Put("labels", m.Labels) + } + key := fmt.Sprintf("%s%s", prettyName.Value, prettyName.Unit) + + var value interface{} + switch v := v.(type) { + case consulDetailedValue: + value = v.Mean + default: + value = v + } + + if _, ok := labels[labelsCombination]; !ok { + temp.Put(key, value) + labels[labelsCombination] = temp + } else { + labels[labelsCombination].Put(key, value) + } +} + +// prettyName is used to translate a name in Consul metrics to a metric name that follows ES naming conventions +// https://www.elastic.co/guide/en/beats/devguide/current/event-conventions.html +func prettyName(s string) *valueHelper { + for k, v := range allowedValues { + if s == k { + return &v + } + } + + for k, v := range allowedDetailedValues { + if s == k { + return &v + } + } + + return nil +} diff --git a/metricbeat/module/consul/agent/data_integration_test.go b/metricbeat/module/consul/agent/data_integration_test.go new file mode 100644 index 000000000000..bfaa35a4f66f --- /dev/null +++ b/metricbeat/module/consul/agent/data_integration_test.go @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package agent + +import ( + "github.com/elastic/beats/metricbeat/module/consul" + "testing" + + _ "github.com/denisenkom/go-mssqldb" + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + t.Skip("Skipping `data.json` generation test") + + f := mbtest.NewReportingMetricSetV2(t, consul.GetConfig([]string{"agent"})) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { + t.Fatal("write", err) + } +} diff --git a/metricbeat/module/consul/agent/input_format.go b/metricbeat/module/consul/agent/input_format.go new file mode 100644 index 000000000000..dfcc0828f6a1 --- /dev/null +++ b/metricbeat/module/consul/agent/input_format.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package agent + +type consulMetric struct { + Name string `json:"Name"` + Labels map[string]string `json:"Labels"` +} + +type gauge consulSimpleValue + +type counter consulDetailedValue + +type sample consulDetailedValue + +type consulSimpleValue struct { + consulMetric + Value float64 `json:"Value"` +} + +type consulDetailedValue struct { + consulMetric + Count int `json:"-"` + Rate float64 `json:"-"` + Sum float64 `json:"-"` + Min float64 `json:"-"` + Max float64 `json:"-"` + Mean float64 `json:"Mean"` + Stddev float64 `json:"-"` +} + +type point consulSimpleValue + +type agent struct { + Timestamp string `json:"Timestamp"` + Gauges []gauge `json:"Gauges"` + Points []point `json:"Points"` + Counters []counter `json:"Counters"` + Samples []sample `json:"Samples"` +} diff --git a/metricbeat/module/consul/fields.go b/metricbeat/module/consul/fields.go new file mode 100644 index 000000000000..bc8dbb85a9d9 --- /dev/null +++ b/metricbeat/module/consul/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package consul + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "consul", asset.ModuleFieldsPri, AssetConsul); err != nil { + panic(err) + } +} + +// AssetConsul returns asset data. +// This is the base64 encoded gzipped contents of ../metricbeat/module/consul. +func AssetConsul() string { + return "eJzcV01v4zYQvftXPOy5MdqrDwXSHtoeugts9x6MpLHMhiKFmWFS/fuClJT4Q7JTJECL8pKAH2/eezND0Xd45GGHOgZNfgOYM887fBonPm2AhrUW15uLYYcfNwDwc1lEF5vkeQMIeyblHSo22gB7x77RXdl7h0AdH0XIw4aed2glpn6aOT5yfIxaDvYyu3Qyj3MC81jgPo/7DIzf2cTVyoY9W31gRTfOwIV9lI7yUewldqBZtgtqFGqGpBBcaEE6oh3hn8s5liS0t5OFNVVrQKe2dp2zB3MdX+y5hnsN+xi/08XlGdrH0K5sOPH+/omFWkbmCRfQOe+dch1Do3AGo0dWWJz0gBD4GRxMhjxrB4YJBaW6JMTHFnFfpj1Tw7LqD/W9H1adWaF/Qv1z6iqWHM7HVieCxg3U5SIoHEgNLhjLE/ntYtIpWeydjx+X+QOTt8P7tH3JafF+wnqxNNbkoSxPLKh9UmPZ4rc98tZxWkHCpaddw8LNzAbVgPtZ6newg1M8O+9RMXKXWcQPI1QYCkQK08mzzd8v2ygpLJT6uoknar+Oh/NtQTmBU6v/Q991WGqJ93daNRh/TLO9VmzBzP903EUZECsjF7gZb7Sc6y9/bFeVduR9rB/qmMJ52d4kdULoV6YesfqTa1MU0Oz/tcruH6bt7yvv85CIoajOEUChgVMQWg65DWaTemHVJPmealxNFmWLb7k2OxpQJVGb7Mu1lC+n/LdKBj3E5BsIW5KQVwhqTM0ANTLGE/nE63a3UWIyFxar4O2iv05fpVe4S6U+UvOv6SzZ+I+30E+lcV7rphpK3UwvgF5izapXUklSUcsPdfSea4uX36dbck/Y/DLC4QVu5ep6q0+SwsfYdEnMopEH/8V1yluWw8w8ekq69Gy5bc8Fk2+lSA9kJU/tBa8D6RiuKRuoX0O95t8x9zqJ8OK9+HYFb4l3HHM1a6chr2RvHjeyWJx6eawFCnF6q920peT//2DK51fR42+XbrwG1GJ/Zwe+e47im/NKy4/TYp5Oz8PpxlAjMW62m78DAAD//x4/izw=" +} diff --git a/metricbeat/module/consul/testing.go b/metricbeat/module/consul/testing.go new file mode 100644 index 000000000000..135d02921298 --- /dev/null +++ b/metricbeat/module/consul/testing.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package consul + +import ( + "fmt" + "os" +) + +//GetConfig returns a config object specific for a Consul module and a provided Metricset in 'ms' +func GetConfig(ms []string) map[string]interface{} { + return map[string]interface{}{ + "module": "consul", + "metricsets": ms, + "hosts": []string{fmt.Sprintf("%s:%s", EnvOr("CONSUL_HOST", "localhost"), EnvOr("CONSUL_PORT", "8500"))}, + } +} + +// EnvOr returns the value of the specified environment variable if it is +// non-empty. Otherwise it return def. +func EnvOr(name, def string) string { + s := os.Getenv(name) + if s == "" { + return def + } + return s +} diff --git a/metricbeat/modules.d/consul.yml.disabled b/metricbeat/modules.d/consul.yml.disabled new file mode 100644 index 000000000000..d9b9dc5085d9 --- /dev/null +++ b/metricbeat/modules.d/consul.yml.disabled @@ -0,0 +1,10 @@ +# Module: consul +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-consul.html + +- module: consul + metricsets: + - agent + enabled: true + period: 10s + hosts: ["localhost:8500"] + diff --git a/metricbeat/tests/system/test_consul.py b/metricbeat/tests/system/test_consul.py new file mode 100644 index 000000000000..5285aae5ddea --- /dev/null +++ b/metricbeat/tests/system/test_consul.py @@ -0,0 +1,61 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + +CONSUL_FIELDS = metricbeat.COMMON_FIELDS + ["consul"] + +# raft fields not included here as it's not consistently returned by Consul +AGENT_FIELDS = [ + "runtime.garbage_collector.pause.current.ns", + "runtime.garbage_collector.pause.total.ns", + "runtime.garbage_collector.runs", + "runtime.alloc.bytes", + "runtime.heap_objects", + "runtime.malloc_count", + "runtime.goroutines", + "runtime.sys.bytes", +] + + +class ConsulAgentTest(metricbeat.BaseTest): + + COMPOSE_SERVICES = ['consul'] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_output(self): + """ + Consul mntr module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "consul", + "metricsets": ["agent"], + "hosts": self.get_hosts(), + "period": "10s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(CONSUL_FIELDS), evt.keys()) + consul_agent = evt["consul"]["agent"] + + consul_agent.pop("raft", None) + consul_agent.pop("autopilot", None) + + print(consul_agent) + self.assertItemsEqual(self.de_dot(AGENT_FIELDS), consul_agent.keys()) + + assert(consul_agent["runtime"]["heap_objects"] > 0) + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [os.getenv('CONSUL_HOST', 'localhost') + ':' + + os.getenv('CONSUL_PORT', '8500')] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 6eed891a52aa..25163bba0c8a 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -163,6 +163,14 @@ metricbeat.modules: hosts: ["localhost:5000"] enabled: true +#-------------------------------- Consul Module -------------------------------- +- module: consul + metricsets: ["agent"] + enabled: false + period: 10s + hosts: ["localhost:8500"] + + #------------------------------ Couchbase Module ------------------------------ - module: couchbase metricsets: ["bucket", "cluster", "node"] From 81f75bf09bdd0a52801882693128486795807230 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 8 Feb 2019 10:59:51 +0100 Subject: [PATCH 02/14] Address comments --- metricbeat/module/consul/agent/agent_test.go | 56 ++++++++++++++++++++ metricbeat/module/consul/agent/data.go | 23 ++++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/consul/agent/agent_test.go b/metricbeat/module/consul/agent/agent_test.go index 9474b63e5992..523c2449b0ff 100644 --- a/metricbeat/module/consul/agent/agent_test.go +++ b/metricbeat/module/consul/agent/agent_test.go @@ -286,3 +286,59 @@ func TestEventMapping(t *testing.T) { assert.True(t, goroutinesFound) assert.True(t, heapObjectsFound) } + +func TestUniqueKeyForLabelMap(t *testing.T) { + input := []map[string]string{ + { + "a": "b", + "g": "h", + "c": "d", + "e": "f", + "i": "j", + }, + { + "a": "b", + "e": "f", + "c": "d", + "g": "h", + "i": "j", + }, + { + "c": "d", + "a": "b", + "g": "h", + "e": "f", + "i": "j", + }, + { + "c": "d", + "e": "f", + "i": "j", + "a": "b", + "g": "h", + }, + { + "e": "f", + "a": "b", + "c": "d", + "g": "h", + "i": "j", + }, + { + "e": "f", + "i": "j", + "c": "d", + "a": "b", + "g": "h", + }, + } + + keys := make([]string, 0) + for _, i := range input { + keys = append(keys, uniqueKeyForLabelMap(i)) + } + + for i := 1; i < len(keys); i++ { + assert.True(t, keys[i-1] == keys[i]) + } +} diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index d6662aba1f2d..fa340b72fea6 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -20,6 +20,7 @@ package agent import ( "encoding/json" "fmt" + "sort" "github.com/elastic/beats/libbeat/common" ) @@ -87,10 +88,7 @@ func metricApply(labels map[string]common.MapStr, m consulMetric, v interface{}) return } - labelsCombination := "" - for k, v := range m.Labels { - labelsCombination = fmt.Sprintf("%s%s%s", labelsCombination, k, v) - } + labelsCombination := uniqueKeyForLabelMap(m.Labels) temp := common.MapStr{} if len(m.Labels) != 0 { @@ -131,3 +129,20 @@ func prettyName(s string) *valueHelper { return nil } + +// Create a simple unique value for a map of labels without using a hash function +func uniqueKeyForLabelMap(m map[string]string) string { + // In labels terms a:b, c:d should be the same than c:d, a:d, order should not affect, that's why we extract labels first and then we order them + labelsArray := make([]string, 0) + for k, v := range m { + labelsArray = append(labelsArray, fmt.Sprintf("%s%s", k, v)) + } + + sort.Strings(labelsArray) + labelsCombination := "" + for _, v := range labelsArray { + labelsArray = append(labelsArray, v) + } + + return labelsCombination +} From a7bdfa079cd5b7660533906eff5562c37ca194b4 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 11:07:48 +0100 Subject: [PATCH 03/14] Format everything again --- metricbeat/module/consul/agent/input_format.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/consul/agent/input_format.go b/metricbeat/module/consul/agent/input_format.go index dfcc0828f6a1..68e31034b267 100644 --- a/metricbeat/module/consul/agent/input_format.go +++ b/metricbeat/module/consul/agent/input_format.go @@ -35,13 +35,13 @@ type consulSimpleValue struct { type consulDetailedValue struct { consulMetric - Count int `json:"-"` - Rate float64 `json:"-"` - Sum float64 `json:"-"` - Min float64 `json:"-"` - Max float64 `json:"-"` + Count int `json:"Count"` + Rate float64 `json:"Rate"` + Sum float64 `json:"Sum"` + Min float64 `json:"Min"` + Max float64 `json:"Max"` Mean float64 `json:"Mean"` - Stddev float64 `json:"-"` + Stddev float64 `json:"Stddev"` } type point consulSimpleValue From 4fcff0d3ffadf07d50ef995d2a5a51252aa09e42 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 11:08:42 +0100 Subject: [PATCH 04/14] Use common.Mapstr to calculate a unique set of values --- metricbeat/module/consul/agent/data.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index fa340b72fea6..01e77e266b42 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -20,8 +20,6 @@ package agent import ( "encoding/json" "fmt" - "sort" - "github.com/elastic/beats/libbeat/common" ) @@ -43,8 +41,8 @@ var ( "consul.runtime.malloc_count": {Value: "runtime.malloc_count"}, } allowedDetailedValues = map[string]valueHelper{ - "consul.raft.apply": {Value: "raft.apply"}, - "consul.raft.commitTime": {Value: "raft.commit_time", Unit: ".ms"}, + //"consul.raft.apply": {Value: "raft.apply"}, + //"consul.raft.commitTime": {Value: "raft.commit_time", Unit: ".ms"}, } ) @@ -132,17 +130,10 @@ func prettyName(s string) *valueHelper { // Create a simple unique value for a map of labels without using a hash function func uniqueKeyForLabelMap(m map[string]string) string { - // In labels terms a:b, c:d should be the same than c:d, a:d, order should not affect, that's why we extract labels first and then we order them - labelsArray := make([]string, 0) + mm := common.MapStr{} for k, v := range m { - labelsArray = append(labelsArray, fmt.Sprintf("%s%s", k, v)) - } - - sort.Strings(labelsArray) - labelsCombination := "" - for _, v := range labelsArray { - labelsArray = append(labelsArray, v) + mm[k] = v } - return labelsCombination + return mm.String() } From dfe2754b3053c2bd8400a3ea023c70cc16d19ed2 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 11:25:09 +0100 Subject: [PATCH 05/14] Remove fields --- metricbeat/module/consul/agent/_meta/fields.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/metricbeat/module/consul/agent/_meta/fields.yml b/metricbeat/module/consul/agent/_meta/fields.yml index 657a52ff7c9b..ab4b3abee8c5 100644 --- a/metricbeat/module/consul/agent/_meta/fields.yml +++ b/metricbeat/module/consul/agent/_meta/fields.yml @@ -4,18 +4,6 @@ description: > Agent Metricset fetches metrics information from a Consul instance running as Agent fields: - - name: raft - type: group - fields: - - name: commit_time - type: group - fields: - - name: ms - type: long - description: Average time in milliseconds it takes to commit a new entry to the transaction log of the leader - - name: apply - type: long - description: Number of logs committed since the last interval. - name: autopilot type: group fields: From 9039c00b623d55850c2359ec18667aa8098b8182 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 12:00:05 +0100 Subject: [PATCH 06/14] Update data.json to reflect fields removed --- .../module/consul/agent/_meta/data.json | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/metricbeat/module/consul/agent/_meta/data.json b/metricbeat/module/consul/agent/_meta/data.json index 86782e37258b..b9446e8c4942 100644 --- a/metricbeat/module/consul/agent/_meta/data.json +++ b/metricbeat/module/consul/agent/_meta/data.json @@ -9,27 +9,21 @@ "autopilot": { "healthy": 1 }, - "raft": { - "apply": 1, - "commit_time": { - "ms": 0.01523400004953146 - } - }, "runtime": { "alloc": { - "bytes": 8543432 + "bytes": 9654896 }, "garbage_collector": { "pause": { "total": { - "ns": 43749010 + "ns": 33816372 } }, - "runs": 101 + "runs": 105 }, - "goroutines": 76, - "heap_objects": 48889, - "malloc_count": 3200832, + "goroutines": 77, + "heap_objects": 51099, + "malloc_count": 2918194, "sys": { "bytes": 74119416 } @@ -48,4 +42,4 @@ "address": "localhost:8500", "type": "consul" } -} +} \ No newline at end of file From 00b6534caf110142e4231356f6b3050aeaece8c4 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 12:01:37 +0100 Subject: [PATCH 07/14] Fields removed from doc too --- metricbeat/docs/fields.asciidoc | 20 -------------------- metricbeat/module/consul/fields.go | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index f669149dd646..bc5b4f28808e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2122,26 +2122,6 @@ Agent Metricset fetches metrics information from a Consul instance running as Ag - -*`consul.agent.raft.commit_time.ms`*:: -+ --- -type: long - -Average time in milliseconds it takes to commit a new entry to the transaction log of the leader - --- - -*`consul.agent.raft.apply`*:: -+ --- -type: long - -Number of logs committed since the last interval. - --- - - *`consul.agent.autopilot.healthy`*:: + -- diff --git a/metricbeat/module/consul/fields.go b/metricbeat/module/consul/fields.go index bc8dbb85a9d9..4930201796b4 100644 --- a/metricbeat/module/consul/fields.go +++ b/metricbeat/module/consul/fields.go @@ -32,5 +32,5 @@ func init() { // AssetConsul returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/consul. func AssetConsul() string { - return "eJzcV01v4zYQvftXPOy5MdqrDwXSHtoeugts9x6MpLHMhiKFmWFS/fuClJT4Q7JTJECL8pKAH2/eezND0Xd45GGHOgZNfgOYM887fBonPm2AhrUW15uLYYcfNwDwc1lEF5vkeQMIeyblHSo22gB7x77RXdl7h0AdH0XIw4aed2glpn6aOT5yfIxaDvYyu3Qyj3MC81jgPo/7DIzf2cTVyoY9W31gRTfOwIV9lI7yUewldqBZtgtqFGqGpBBcaEE6oh3hn8s5liS0t5OFNVVrQKe2dp2zB3MdX+y5hnsN+xi/08XlGdrH0K5sOPH+/omFWkbmCRfQOe+dch1Do3AGo0dWWJz0gBD4GRxMhjxrB4YJBaW6JMTHFnFfpj1Tw7LqD/W9H1adWaF/Qv1z6iqWHM7HVieCxg3U5SIoHEgNLhjLE/ntYtIpWeydjx+X+QOTt8P7tH3JafF+wnqxNNbkoSxPLKh9UmPZ4rc98tZxWkHCpaddw8LNzAbVgPtZ6newg1M8O+9RMXKXWcQPI1QYCkQK08mzzd8v2ygpLJT6uoknar+Oh/NtQTmBU6v/Q991WGqJ93daNRh/TLO9VmzBzP903EUZECsjF7gZb7Sc6y9/bFeVduR9rB/qmMJ52d4kdULoV6YesfqTa1MU0Oz/tcruH6bt7yvv85CIoajOEUChgVMQWg65DWaTemHVJPmealxNFmWLb7k2OxpQJVGb7Mu1lC+n/LdKBj3E5BsIW5KQVwhqTM0ANTLGE/nE63a3UWIyFxar4O2iv05fpVe4S6U+UvOv6SzZ+I+30E+lcV7rphpK3UwvgF5izapXUklSUcsPdfSea4uX36dbck/Y/DLC4QVu5ep6q0+SwsfYdEnMopEH/8V1yluWw8w8ekq69Gy5bc8Fk2+lSA9kJU/tBa8D6RiuKRuoX0O95t8x9zqJ8OK9+HYFb4l3HHM1a6chr2RvHjeyWJx6eawFCnF6q920peT//2DK51fR42+XbrwG1GJ/Zwe+e47im/NKy4/TYp5Oz8PpxlAjMW62m78DAAD//x4/izw=" + return "eJzcVsGO4zYMvecrHvbcBO01hwLbHtoeugts9z6gJSZWK4uGRM3Wf19Ijnccx85k0AItqssMJPHxvUeK8R5/8HCEkZCy3wHq1PMR78aNdzvAcjLR9eokHPH9DgB+rIfoxGbPOyCyZ0p8RMNKO+Dk2Nt0rHf3CNTxLENZOvR8xDlK7i8785B5GJ056NfdtciylgSmtcJ9Wu8LMH5ljc4kVpxYTcsJ3bgDF04SOyqhOEXpQJNsF5JSMIyYQ3DhDEoj2gx/KedKUlbpnRe9Ot2StoU2R2yZvLbDzfmE6SWcVw6v3Pn4zJG8v2BBTtCW4cWQR+L4zBHG56QcD/jlhHJ13E6gyLW8znJkO7FBM+D9JPUbaOsSvjjv0TCK4Sr4boQKQ4XI4RK5uPztYdXGmIO6jh828UrtpzG4NA4p26nqb/Q9DcuI+xTuYc5xm0F5DRmvVfRG54fcNRxLNStm+afjTuIAaZRcYDs2d6n1x98Om0o78l7Mk5Eclm37KqkrQj8z9ZDmdzaaUEGL//c6u3+6XP977b1MCQlVdckAChYugXDmUJ7BZFIfOaUcGS5YZ0glHvC59GZHA5ock17sK72kMv5tsiK1kr1FZM0xlBNCUiY7ICkp45l85m27zxIlqwurXfC46E+XAfUCd6vUC9l/TWetxn/8Cf1QH85L3zRD7ZvLj0EfxXBKd0pJsaEzPxnxno1KfLPcKzY/jXD4Crcxuh71Kebwz9h0S0xFyYP/ZJPLlfU0E4+eclrO8sfsuWHyuTZpS1rrdL7h1VIa09l6gfot1Hv+zbmbHCOvzsXHFTySb55zs2rXKe9Ub1qvVLE6NT54FxAoSGIjwW7nnyjW+v8fTPnwInr8jO3GMZBU+r22vP8i0dtlp5UPx2peQnLlc/EyMZJSVLaH3V8BAAD//w8tCWs=" } From 2c8f20a95af6c4d014332bc44c8bce9f5f0f9de6 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 12:12:12 +0100 Subject: [PATCH 08/14] make fmt --- metricbeat/module/consul/agent/data.go | 1 + metricbeat/module/consul/agent/data_integration_test.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index 01e77e266b42..2d8721ec9e03 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -20,6 +20,7 @@ package agent import ( "encoding/json" "fmt" + "github.com/elastic/beats/libbeat/common" ) diff --git a/metricbeat/module/consul/agent/data_integration_test.go b/metricbeat/module/consul/agent/data_integration_test.go index bfaa35a4f66f..bffe36098be9 100644 --- a/metricbeat/module/consul/agent/data_integration_test.go +++ b/metricbeat/module/consul/agent/data_integration_test.go @@ -20,9 +20,10 @@ package agent import ( - "github.com/elastic/beats/metricbeat/module/consul" "testing" + "github.com/elastic/beats/metricbeat/module/consul" + _ "github.com/denisenkom/go-mssqldb" "github.com/stretchr/testify/assert" From c86cd3eb82e4ea38cf87d7e839646a5d205982ce Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 13:22:04 +0100 Subject: [PATCH 09/14] Set autopilot.healthy to boolean. Chage fields and run mage fmt update --- metricbeat/docs/fields.asciidoc | 4 +- .../module/consul/agent/_meta/data.json | 17 ++-- .../module/consul/agent/_meta/fields.yml | 4 +- metricbeat/module/consul/agent/data.go | 85 ++++++++++++++----- metricbeat/module/consul/fields.go | 2 +- x-pack/metricbeat/metricbeat.reference.yml | 5 +- 6 files changed, 82 insertions(+), 35 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index bc5b4f28808e..42ba7d7799ce 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2125,9 +2125,9 @@ Agent Metricset fetches metrics information from a Consul instance running as Ag *`consul.agent.autopilot.healthy`*:: + -- -type: long +type: boolean -Overall health of the local server cluster. If all servers are considered healthy by Autopilot, this will be set to 1. If any are unhealthy, this will be 0. +Overall health of the local server cluster -- diff --git a/metricbeat/module/consul/agent/_meta/data.json b/metricbeat/module/consul/agent/_meta/data.json index b9446e8c4942..f1e476c23fc3 100644 --- a/metricbeat/module/consul/agent/_meta/data.json +++ b/metricbeat/module/consul/agent/_meta/data.json @@ -7,23 +7,28 @@ "consul": { "agent": { "autopilot": { - "healthy": 1 + "healthy": true + }, + "raft": { + "commit_time": { + "ms": 0.04560700058937073 + } }, "runtime": { "alloc": { - "bytes": 9654896 + "bytes": 6900784 }, "garbage_collector": { "pause": { "total": { - "ns": 33816372 + "ns": 99490250 } }, - "runs": 105 + "runs": 287 }, "goroutines": 77, - "heap_objects": 51099, - "malloc_count": 2918194, + "heap_objects": 42590, + "malloc_count": 7087169, "sys": { "bytes": 74119416 } diff --git a/metricbeat/module/consul/agent/_meta/fields.yml b/metricbeat/module/consul/agent/_meta/fields.yml index ab4b3abee8c5..bc95fffe4e57 100644 --- a/metricbeat/module/consul/agent/_meta/fields.yml +++ b/metricbeat/module/consul/agent/_meta/fields.yml @@ -8,8 +8,8 @@ type: group fields: - name: healthy - type: long - description: Overall health of the local server cluster. If all servers are considered healthy by Autopilot, this will be set to 1. If any are unhealthy, this will be 0. + type: boolean + description: Overall health of the local server cluster - name: runtime type: group description: Runtime related metrics diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index 2d8721ec9e03..6d2ef18700e7 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -24,26 +24,68 @@ import ( "github.com/elastic/beats/libbeat/common" ) +type valueConverter interface { + GetValue(i interface{}) interface{} +} + +type keyRenamer interface { + Key() string +} + +type inputConverter interface { + valueConverter + keyRenamer +} + type valueHelper struct { - Value string - Unit string + renamedTo string + unit string +} + +func (v *valueHelper) Key() string { + if v.unit == "" { + return v.renamedTo + } + + return fmt.Sprintf("%s.%s", v.renamedTo, v.unit) +} + +type convertToBooleanValue struct { + valueHelper +} + +func (v *convertToBooleanValue) GetValue(i interface{}) interface{} { + value, ok := i.(float64) + if !ok { + return nil + } + + return value == 1 +} + +type noConversionValue struct { + valueHelper +} + +func (v *noConversionValue) GetValue(i interface{}) interface{} { + return i } var ( - allowedValues = map[string]valueHelper{ - "consul.autopilot.healthy": {Value: "autopilot.healthy"}, - "consul.runtime.alloc_bytes": {Value: "runtime.alloc", Unit: ".bytes"}, - "consul.runtime.total_gc_pause_ns": {Value: "runtime.garbage_collector.pause.total", Unit: ".ns"}, - "consul.runtime.gc_pause_ns": {Value: "runtime.garbage_collector.pause.current", Unit: ".ns"}, - "consul.runtime.total_gc_runs": {Value: "runtime.garbage_collector.runs"}, - "consul.runtime.num_goroutines": {Value: "runtime.goroutines"}, - "consul.runtime.heap_objects": {Value: "runtime.heap_objects"}, - "consul.runtime.sys_bytes": {Value: "runtime.sys", Unit: ".bytes"}, - "consul.runtime.malloc_count": {Value: "runtime.malloc_count"}, - } - allowedDetailedValues = map[string]valueHelper{ - //"consul.raft.apply": {Value: "raft.apply"}, - //"consul.raft.commitTime": {Value: "raft.commit_time", Unit: ".ms"}, + allowedValues = map[string]inputConverter{ + "consul.autopilot.healthy": &convertToBooleanValue{valueHelper{renamedTo: "autopilot.healthy"}}, + "consul.runtime.alloc_bytes": &noConversionValue{valueHelper{renamedTo: "runtime.alloc", unit: "bytes"}}, + "consul.runtime.total_gc_pause_ns": &noConversionValue{valueHelper{renamedTo: "runtime.garbage_collector.pause.total", unit: "ns"}}, + "consul.runtime.gc_pause_ns": &noConversionValue{valueHelper{renamedTo: "runtime.garbage_collector.pause.current", unit: "ns"}}, + "consul.runtime.total_gc_runs": &noConversionValue{valueHelper{renamedTo: "runtime.garbage_collector.runs"}}, + "consul.runtime.num_goroutines": &noConversionValue{valueHelper{renamedTo: "runtime.goroutines"}}, + "consul.runtime.heap_objects": &noConversionValue{valueHelper{renamedTo: "runtime.heap_objects"}}, + "consul.runtime.sys_bytes": &noConversionValue{valueHelper{renamedTo: "runtime.sys", unit: "bytes"}}, + "consul.runtime.malloc_count": &noConversionValue{valueHelper{renamedTo: "runtime.malloc_count"}}, + } + allowedDetailedValues = map[string]inputConverter{ + "consul.raft.apply": &noConversionValue{valueHelper{renamedTo: "raft.apply"}}, + "consul.raft.commitTime": &noConversionValue{valueHelper{renamedTo: "raft.commit_time", unit: "ms"}}, } ) @@ -93,7 +135,6 @@ func metricApply(labels map[string]common.MapStr, m consulMetric, v interface{}) if len(m.Labels) != 0 { temp.Put("labels", m.Labels) } - key := fmt.Sprintf("%s%s", prettyName.Value, prettyName.Unit) var value interface{} switch v := v.(type) { @@ -104,25 +145,25 @@ func metricApply(labels map[string]common.MapStr, m consulMetric, v interface{}) } if _, ok := labels[labelsCombination]; !ok { - temp.Put(key, value) + temp.Put(prettyName.Key(), prettyName.GetValue(value)) labels[labelsCombination] = temp } else { - labels[labelsCombination].Put(key, value) + labels[labelsCombination].Put(prettyName.Key(), prettyName.GetValue(value)) } } // prettyName is used to translate a name in Consul metrics to a metric name that follows ES naming conventions // https://www.elastic.co/guide/en/beats/devguide/current/event-conventions.html -func prettyName(s string) *valueHelper { +func prettyName(s string) inputConverter { for k, v := range allowedValues { if s == k { - return &v + return v } } for k, v := range allowedDetailedValues { if s == k { - return &v + return v } } diff --git a/metricbeat/module/consul/fields.go b/metricbeat/module/consul/fields.go index 4930201796b4..5bd61822219a 100644 --- a/metricbeat/module/consul/fields.go +++ b/metricbeat/module/consul/fields.go @@ -32,5 +32,5 @@ func init() { // AssetConsul returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/consul. func AssetConsul() string { - return "eJzcVsGO4zYMvecrHvbcBO01hwLbHtoeugts9z6gJSZWK4uGRM3Wf19Ijnccx85k0AItqssMJPHxvUeK8R5/8HCEkZCy3wHq1PMR78aNdzvAcjLR9eokHPH9DgB+rIfoxGbPOyCyZ0p8RMNKO+Dk2Nt0rHf3CNTxLENZOvR8xDlK7i8785B5GJ056NfdtciylgSmtcJ9Wu8LMH5ljc4kVpxYTcsJ3bgDF04SOyqhOEXpQJNsF5JSMIyYQ3DhDEoj2gx/KedKUlbpnRe9Ot2StoU2R2yZvLbDzfmE6SWcVw6v3Pn4zJG8v2BBTtCW4cWQR+L4zBHG56QcD/jlhHJ13E6gyLW8znJkO7FBM+D9JPUbaOsSvjjv0TCK4Sr4boQKQ4XI4RK5uPztYdXGmIO6jh828UrtpzG4NA4p26nqb/Q9DcuI+xTuYc5xm0F5DRmvVfRG54fcNRxLNStm+afjTuIAaZRcYDs2d6n1x98Om0o78l7Mk5Eclm37KqkrQj8z9ZDmdzaaUEGL//c6u3+6XP977b1MCQlVdckAChYugXDmUJ7BZFIfOaUcGS5YZ0glHvC59GZHA5ock17sK72kMv5tsiK1kr1FZM0xlBNCUiY7ICkp45l85m27zxIlqwurXfC46E+XAfUCd6vUC9l/TWetxn/8Cf1QH85L3zRD7ZvLj0EfxXBKd0pJsaEzPxnxno1KfLPcKzY/jXD4Crcxuh71Kebwz9h0S0xFyYP/ZJPLlfU0E4+eclrO8sfsuWHyuTZpS1rrdL7h1VIa09l6gfot1Hv+zbmbHCOvzsXHFTySb55zs2rXKe9Ub1qvVLE6NT54FxAoSGIjwW7nnyjW+v8fTPnwInr8jO3GMZBU+r22vP8i0dtlp5UPx2peQnLlc/EyMZJSVLaH3V8BAAD//w8tCWs=" + return "eJzcVsFu2zAMvecriN7TD8hhwLbDdlkLdL0XtMzY2mTRIKlu/vtBdtzYiZ2m2IAN0yWBJD6+90gT2sJ36nbgOGoKGwDzFmgHN8PGzQagJHXiW/Mcd/BuAwDwsT+EhssUaAMgFAiVdlCQ4QZg7ymUuuvvbiFiQ5MMeVnX0g4q4dQedqYh0zCsKNrL7lJkXqcExrXAfVzvMzB8IRPvlAz2ZK4mhWbYAR/3LA3mUNgLN4CjbB/VMDoCSTH6WAHqgDbBP5Uzk5SMWx/YZqdr0tbQpog1YbC6OzsfMQvmQBgXzmcG3T+TYAgHOOA9WE0Q2GEAJXkmAReSGsmiLknRfENXq5rlfhiCcyXRqBzL8EYjtDuNuEzhEuYUt+iMlpCP6IFjtXJhpvMuNQVJ9rbHzH8aalg64MLQRyqHbsvO33+9XVXaYAjsnhyneNpHr5KaEfpM2AIX38iZQg+a/b/Uau3T4fpv5b0/TQkce9U5A2AswSsgVBRzU44mtUKqSQh8LL1DY7mFx9orNNhBkUTtYF/uJePht0gGWnMKJQhZkphPENQIyw7U0AieMSRat7ti4WQ+LnbB9aIfDhPjCHeuNDCWf01nX41//BP60H84x74pur5vDtO5FXakeqGUKAVW9OQ4BHLG8ma5MzafBjh4gVsZXdf6JCn+GZvOiRkbBqCf5FK+spxm5NFi0tNZfp09Z0we+yat0fo6VWe8atQhXdlfwHYN9ZJ/U+4uidDiXLxewTX5pjlXqzZPeaF643qlir1TwwfvI0SMrOQ4luv5R4p9/f8HU+6Oood3ZTOMATVut1bT9gdLKE87Lb/kevMU1Of322FiqKEYlbebXwEAAP//P1vjFg==" } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 25163bba0c8a..68362dd1147f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -165,8 +165,9 @@ metricbeat.modules: #-------------------------------- Consul Module -------------------------------- - module: consul - metricsets: ["agent"] - enabled: false + metricsets: + - agent + enabled: true period: 10s hosts: ["localhost:8500"] From 4fc3681e6167c8224f50c39db6d873834c61768c Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 16:06:38 +0100 Subject: [PATCH 10/14] Comment again removed fields --- metricbeat/module/consul/agent/data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index 6d2ef18700e7..7f91b6f7a9b6 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -84,8 +84,8 @@ var ( "consul.runtime.malloc_count": &noConversionValue{valueHelper{renamedTo: "runtime.malloc_count"}}, } allowedDetailedValues = map[string]inputConverter{ - "consul.raft.apply": &noConversionValue{valueHelper{renamedTo: "raft.apply"}}, - "consul.raft.commitTime": &noConversionValue{valueHelper{renamedTo: "raft.commit_time", unit: "ms"}}, + //"consul.raft.apply": &noConversionValue{valueHelper{renamedTo: "raft.apply"}}, + //"consul.raft.commitTime": &noConversionValue{valueHelper{renamedTo: "raft.commit_time", unit: "ms"}}, } ) From 8d851e4efda4e2d7c4834f12858242fdca458e27 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 16:13:01 +0100 Subject: [PATCH 11/14] Better naming of some types --- metricbeat/module/consul/agent/data.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index 7f91b6f7a9b6..282705b2b89d 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -25,11 +25,11 @@ import ( ) type valueConverter interface { - GetValue(i interface{}) interface{} + Convert(i interface{}) interface{} } type keyRenamer interface { - Key() string + Rename() string } type inputConverter interface { @@ -42,7 +42,7 @@ type valueHelper struct { unit string } -func (v *valueHelper) Key() string { +func (v *valueHelper) Rename() string { if v.unit == "" { return v.renamedTo } @@ -50,11 +50,11 @@ func (v *valueHelper) Key() string { return fmt.Sprintf("%s.%s", v.renamedTo, v.unit) } -type convertToBooleanValue struct { +type boolValue struct { valueHelper } -func (v *convertToBooleanValue) GetValue(i interface{}) interface{} { +func (v *boolValue) Convert(i interface{}) interface{} { value, ok := i.(float64) if !ok { return nil @@ -67,13 +67,13 @@ type noConversionValue struct { valueHelper } -func (v *noConversionValue) GetValue(i interface{}) interface{} { +func (v *noConversionValue) Convert(i interface{}) interface{} { return i } var ( allowedValues = map[string]inputConverter{ - "consul.autopilot.healthy": &convertToBooleanValue{valueHelper{renamedTo: "autopilot.healthy"}}, + "consul.autopilot.healthy": &boolValue{valueHelper{renamedTo: "autopilot.healthy"}}, "consul.runtime.alloc_bytes": &noConversionValue{valueHelper{renamedTo: "runtime.alloc", unit: "bytes"}}, "consul.runtime.total_gc_pause_ns": &noConversionValue{valueHelper{renamedTo: "runtime.garbage_collector.pause.total", unit: "ns"}}, "consul.runtime.gc_pause_ns": &noConversionValue{valueHelper{renamedTo: "runtime.garbage_collector.pause.current", unit: "ns"}}, @@ -145,10 +145,10 @@ func metricApply(labels map[string]common.MapStr, m consulMetric, v interface{}) } if _, ok := labels[labelsCombination]; !ok { - temp.Put(prettyName.Key(), prettyName.GetValue(value)) + temp.Put(prettyName.Rename(), prettyName.Convert(value)) labels[labelsCombination] = temp } else { - labels[labelsCombination].Put(prettyName.Key(), prettyName.GetValue(value)) + labels[labelsCombination].Put(prettyName.Rename(), prettyName.Convert(value)) } } From 4da09c7c5ae05e92cf04e1fd4c23ac0b372a4cfa Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 11 Feb 2019 19:51:34 +0100 Subject: [PATCH 12/14] Remove commented code --- metricbeat/module/consul/agent/data.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metricbeat/module/consul/agent/data.go b/metricbeat/module/consul/agent/data.go index 282705b2b89d..022e066b861f 100644 --- a/metricbeat/module/consul/agent/data.go +++ b/metricbeat/module/consul/agent/data.go @@ -83,10 +83,7 @@ var ( "consul.runtime.sys_bytes": &noConversionValue{valueHelper{renamedTo: "runtime.sys", unit: "bytes"}}, "consul.runtime.malloc_count": &noConversionValue{valueHelper{renamedTo: "runtime.malloc_count"}}, } - allowedDetailedValues = map[string]inputConverter{ - //"consul.raft.apply": &noConversionValue{valueHelper{renamedTo: "raft.apply"}}, - //"consul.raft.commitTime": &noConversionValue{valueHelper{renamedTo: "raft.commit_time", unit: "ms"}}, - } + allowedDetailedValues = map[string]inputConverter{} ) func eventMapping(content []byte) ([]common.MapStr, error) { From 91b2e1ff0b887bfae555f2c3682bc0725cae515f Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 13 Feb 2019 11:54:05 +0100 Subject: [PATCH 13/14] Fix wrong comment --- metricbeat/tests/system/test_consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_consul.py b/metricbeat/tests/system/test_consul.py index 5285aae5ddea..dc7bcbad1d37 100644 --- a/metricbeat/tests/system/test_consul.py +++ b/metricbeat/tests/system/test_consul.py @@ -26,7 +26,7 @@ class ConsulAgentTest(metricbeat.BaseTest): @attr('integration') def test_output(self): """ - Consul mntr module outputs an event. + Consul agent module outputs an event. """ self.render_config_template(modules=[{ "name": "consul", From 531f66d0eb203edd4448cfc380d7c6a6409d67b9 Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 13 Feb 2019 13:05:07 +0100 Subject: [PATCH 14/14] Raise timeout of Consul module to try to fix the test --- metricbeat/tests/system/test_consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_consul.py b/metricbeat/tests/system/test_consul.py index dc7bcbad1d37..374ec6b8fe1e 100644 --- a/metricbeat/tests/system/test_consul.py +++ b/metricbeat/tests/system/test_consul.py @@ -35,7 +35,7 @@ def test_output(self): "period": "10s" }]) proc = self.start_beat() - self.wait_until(lambda: self.output_lines() > 0) + self.wait_until(lambda: self.output_lines() > 0, max_timeout=30) proc.check_kill_and_wait() self.assert_no_logged_warnings()