From d2e3fcd392011c568d5df5704b0535fa7ab7a56b Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 15 Dec 2016 16:00:30 +0100 Subject: [PATCH] Add Prometheus module to Metricbeat This first version implements very basic stats from the prometheus service itself. --- CHANGELOG.asciidoc | 2 +- metricbeat/_meta/beat.full.yml | 7 ++ metricbeat/docker-compose.yml | 14 +++- metricbeat/docker-entrypoint.sh | 1 + metricbeat/docs/fields.asciidoc | 61 ++++++++++++++ metricbeat/docs/modules/prometheus.asciidoc | 36 ++++++++ .../docs/modules/prometheus/stats.asciidoc | 19 +++++ metricbeat/docs/modules_list.asciidoc | 2 + metricbeat/include/list.go | 2 + metricbeat/metricbeat.full.yml | 7 ++ metricbeat/metricbeat.template-es2x.json | 32 +++++++ metricbeat/metricbeat.template.json | 32 +++++++ metricbeat/module/prometheus/_meta/config.yml | 5 ++ .../module/prometheus/_meta/docs.asciidoc | 4 + metricbeat/module/prometheus/_meta/fields.yml | 12 +++ metricbeat/module/prometheus/doc.go | 4 + .../module/prometheus/stats/_meta/data.json | 28 +++++++ .../prometheus/stats/_meta/docs.asciidoc | 3 + .../module/prometheus/stats/_meta/fields.yml | 26 ++++++ metricbeat/module/prometheus/stats/data.go | 26 ++++++ metricbeat/module/prometheus/stats/stats.go | 84 +++++++++++++++++++ .../stats/stats_integration_test.go | 62 ++++++++++++++ metricbeat/tests/system/test_prometheus.py | 38 +++++++++ 23 files changed, 503 insertions(+), 4 deletions(-) create mode 100644 metricbeat/docs/modules/prometheus.asciidoc create mode 100644 metricbeat/docs/modules/prometheus/stats.asciidoc create mode 100644 metricbeat/module/prometheus/_meta/config.yml create mode 100644 metricbeat/module/prometheus/_meta/docs.asciidoc create mode 100644 metricbeat/module/prometheus/_meta/fields.yml create mode 100644 metricbeat/module/prometheus/doc.go create mode 100644 metricbeat/module/prometheus/stats/_meta/data.json create mode 100644 metricbeat/module/prometheus/stats/_meta/docs.asciidoc create mode 100644 metricbeat/module/prometheus/stats/_meta/fields.yml create mode 100644 metricbeat/module/prometheus/stats/data.go create mode 100644 metricbeat/module/prometheus/stats/stats.go create mode 100644 metricbeat/module/prometheus/stats/stats_integration_test.go create mode 100644 metricbeat/tests/system/test_prometheus.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2abd530d3a7c..afd4d70dce77 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -55,7 +55,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Kafka module broker matching enhancements. {pull}3129[3129] - Add a couchbase module with metricsets for node, cluster and bucker. {pull}3081[3081] - Export number of cores for cpu module. {pull}3192[3192] - +- Experimental Prometheus module. {pull}3202[3202] *Packetbeat* diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index 927307d31631..daa928dfa3a2 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -202,6 +202,13 @@ metricbeat.modules: #password: pass +#----------------------------- Prometheus Module ----------------------------- +#- module: prometheus + #metricsets: ["stats"] + #enabled: true + #period: 10s + #hosts: ["localhost:9090"] + #-------------------------------- Redis Module ------------------------------- #- module: redis #metricsets: ["info", "keyspace"] diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index baa670e42e78..8b56aca804d8 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -6,19 +6,22 @@ services: - apache - couchbase - mongodb + - haproxy - kafka - mysql - nginx - postgresql + - prometheus - redis - zookeeper - - haproxy environment: - APACHE_HOST=apache - APACHE_PORT=80 - COUCHBASE_HOST=couchbase - COUCHBASE_PORT=8091 - COUCHBASE_DSN=http://Administrator:password@couchbase:8091 + - HAPROXY_HOST=haproxy + - HAPROXY_PORT=14567 - KAFKA_HOST=kafka - KAFKA_PORT=9092 - NGINX_HOST=nginx @@ -34,10 +37,10 @@ services: - POSTGRESQL_HOST=postgresql - POSTGRESQL_PORT=5432 - POSTGRESQL_USERNAME=postgres + - PROMETHEUS_HOST=prometheus + - PROMETHEUS_PORT=9090 - ZOOKEEPER_HOST=zookeeper - ZOOKEEPER_PORT=2181 - - HAPROXY_HOST=haproxy - - HAPROXY_PORT=14567 - TEST_ENVIRONMENT=false working_dir: /go/src/github.com/elastic/beats/metricbeat volumes: @@ -82,6 +85,11 @@ services: postgresql: image: postgres:9.5.3 + prometheus: + image: prom/prometheus + expose: + - 9090 + redis: image: redis:3.2.4-alpine diff --git a/metricbeat/docker-entrypoint.sh b/metricbeat/docker-entrypoint.sh index 0d3020d51868..25087fc7fa36 100755 --- a/metricbeat/docker-entrypoint.sh +++ b/metricbeat/docker-entrypoint.sh @@ -29,6 +29,7 @@ waitFor ${KAFKA_HOST} ${KAFKA_PORT} Kafka waitFor ${MYSQL_HOST} ${MYSQL_PORT} MySQL waitFor ${NGINX_HOST} ${NGINX_PORT} Nginx waitFor ${POSTGRESQL_HOST} ${POSTGRESQL_PORT} Postgresql +waitFor ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} Prometheus waitFor ${REDIS_HOST} ${REDIS_PORT} Redis waitFor ${ZOOKEEPER_HOST} ${ZOOKEEPER_PORT} Zookeeper exec "$@" diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c338ac5e0a41..cbae706a2682 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -24,6 +24,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3956,6 +3957,66 @@ type: date Time at which these statistics were last reset. +[[exported-fields-prometheus]] +== Prometheus Fields + +prometheus Module +experimental[] + + + +[float] +== prometheus Fields + + + + +[float] +== stats Fields + +stats + + + +[float] +== notifications Fields + +Notification stats + + + +[float] +=== prometheus.stats.notifications.queue_length + +type: long + +Current queue length + + +[float] +=== prometheus.stats.notifications.dropped + +type: long + +Dropped queue events + + +[float] +=== prometheus.stats.processes.open_fds + +type: long + +Open file descriptors gauge + + +[float] +=== prometheus.stats.storage.chunks_to_persist + +type: long + +Gauge on chunks which are not persisted to disk yet + + [[exported-fields-redis]] == Redis Fields diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc new file mode 100644 index 000000000000..950451af00bf --- /dev/null +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -0,0 +1,36 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-prometheus]] +== prometheus Module + +This is the prometheus Module. + + + +[float] +=== Example Configuration + +The Prometheus module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +#- module: prometheus + #metricsets: ["stats"] + #enabled: true + #period: 10s + #hosts: ["localhost:9090"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::prometheus/stats.asciidoc[] + diff --git a/metricbeat/docs/modules/prometheus/stats.asciidoc b/metricbeat/docs/modules/prometheus/stats.asciidoc new file mode 100644 index 000000000000..10aa159ca967 --- /dev/null +++ b/metricbeat/docs/modules/prometheus/stats.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-prometheus-stats]] +include::../../../module/prometheus/stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/prometheus/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index a7c3f55ef9f2..29e714950a18 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -11,6 +11,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -27,6 +28,7 @@ include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] include::modules/postgresql.asciidoc[] +include::modules/prometheus.asciidoc[] include::modules/redis.asciidoc[] include::modules/system.asciidoc[] include::modules/zookeeper.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 4485239cc6cc..be3c90f8f45a 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -36,6 +36,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/postgresql/activity" _ "github.com/elastic/beats/metricbeat/module/postgresql/bgwriter" _ "github.com/elastic/beats/metricbeat/module/postgresql/database" + _ "github.com/elastic/beats/metricbeat/module/prometheus" + _ "github.com/elastic/beats/metricbeat/module/prometheus/stats" _ "github.com/elastic/beats/metricbeat/module/redis" _ "github.com/elastic/beats/metricbeat/module/redis/info" _ "github.com/elastic/beats/metricbeat/module/redis/keyspace" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 751fa4f40c95..b963a4fad372 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -202,6 +202,13 @@ metricbeat.modules: #password: pass +#----------------------------- Prometheus Module ----------------------------- +#- module: prometheus + #metricsets: ["stats"] + #enabled: true + #period: 10s + #hosts: ["localhost:9090"] + #-------------------------------- Redis Module ------------------------------- #- module: redis #metricsets: ["info", "keyspace"] diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 7f5df5604cda..e8e48d31b51c 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2225,6 +2225,38 @@ } } }, + "prometheus": { + "properties": { + "stats": { + "properties": { + "notifications": { + "properties": { + "dropped": { + "type": "long" + }, + "queue_length": { + "type": "long" + } + } + }, + "processes": { + "properties": { + "open_fds": { + "type": "long" + } + } + }, + "storage": { + "properties": { + "chunks_to_persist": { + "type": "long" + } + } + } + } + } + } + }, "redis": { "properties": { "info": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 921703077041..aca7dcb63525 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2210,6 +2210,38 @@ } } }, + "prometheus": { + "properties": { + "stats": { + "properties": { + "notifications": { + "properties": { + "dropped": { + "type": "long" + }, + "queue_length": { + "type": "long" + } + } + }, + "processes": { + "properties": { + "open_fds": { + "type": "long" + } + } + }, + "storage": { + "properties": { + "chunks_to_persist": { + "type": "long" + } + } + } + } + } + } + }, "redis": { "properties": { "info": { diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml new file mode 100644 index 000000000000..f43dad99d547 --- /dev/null +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -0,0 +1,5 @@ +#- module: prometheus + #metricsets: ["stats"] + #enabled: true + #period: 10s + #hosts: ["localhost:9090"] diff --git a/metricbeat/module/prometheus/_meta/docs.asciidoc b/metricbeat/module/prometheus/_meta/docs.asciidoc new file mode 100644 index 000000000000..8ad5916bb908 --- /dev/null +++ b/metricbeat/module/prometheus/_meta/docs.asciidoc @@ -0,0 +1,4 @@ +== prometheus Module + +This is the prometheus Module. + diff --git a/metricbeat/module/prometheus/_meta/fields.yml b/metricbeat/module/prometheus/_meta/fields.yml new file mode 100644 index 000000000000..b3af660a692f --- /dev/null +++ b/metricbeat/module/prometheus/_meta/fields.yml @@ -0,0 +1,12 @@ +- key: prometheus + title: "Prometheus" + description: > + prometheus Module + + experimental[] + short_config: false + fields: + - name: prometheus + type: group + description: > + fields: diff --git a/metricbeat/module/prometheus/doc.go b/metricbeat/module/prometheus/doc.go new file mode 100644 index 000000000000..1324b9559966 --- /dev/null +++ b/metricbeat/module/prometheus/doc.go @@ -0,0 +1,4 @@ +/* +Package prometheus is a Metricbeat module that contains MetricSets. +*/ +package prometheus diff --git a/metricbeat/module/prometheus/stats/_meta/data.json b/metricbeat/module/prometheus/stats/_meta/data.json new file mode 100644 index 000000000000..22c8d3519fda --- /dev/null +++ b/metricbeat/module/prometheus/stats/_meta/data.json @@ -0,0 +1,28 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "127.0.0.1:9090", + "module": "prometheus", + "name": "stats", + "rtt": 115 + }, + "prometheus": { + "stats": { + "notifications": { + "dropped": 0, + "queue_length": 0 + }, + "processes": { + "open_fds": 24 + }, + "storage": { + "chunks_to_persist": 465 + } + } + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/prometheus/stats/_meta/docs.asciidoc b/metricbeat/module/prometheus/stats/_meta/docs.asciidoc new file mode 100644 index 000000000000..ff8442eb02fe --- /dev/null +++ b/metricbeat/module/prometheus/stats/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== prometheus stats MetricSet + +This is the stats metricset of the module prometheus. diff --git a/metricbeat/module/prometheus/stats/_meta/fields.yml b/metricbeat/module/prometheus/stats/_meta/fields.yml new file mode 100644 index 000000000000..72c309a69386 --- /dev/null +++ b/metricbeat/module/prometheus/stats/_meta/fields.yml @@ -0,0 +1,26 @@ +- name: stats + type: group + description: > + stats + fields: + - name: notifications + type: group + description: > + Notification stats + fields: + - name: queue_length + type: long + description: > + Current queue length + - name: dropped + type: long + description: > + Dropped queue events + - name: processes.open_fds + type: long + description: > + Open file descriptors gauge + - name: storage.chunks_to_persist + type: long + description: > + Gauge on chunks which are not persisted to disk yet diff --git a/metricbeat/module/prometheus/stats/data.go b/metricbeat/module/prometheus/stats/data.go new file mode 100644 index 000000000000..f1ea2a02a978 --- /dev/null +++ b/metricbeat/module/prometheus/stats/data.go @@ -0,0 +1,26 @@ +package stats + +import ( + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstrstr" +) + +var ( + schema = s.Schema{ + "notifications": s.Object{ + "queue_length": c.Int("prometheus_notifications_queue_length"), + "dropped": c.Int("prometheus_notifications_dropped_total"), + }, + "processes": s.Object{ + "open_fds": c.Int("process_open_fds"), + }, + "storage": s.Object{ + "chunks_to_persist": c.Int("prometheus_local_storage_chunks_to_persist"), + }, + } +) + +func eventMapping(entries map[string]interface{}) (common.MapStr, error) { + return schema.Apply(entries), nil +} diff --git a/metricbeat/module/prometheus/stats/stats.go b/metricbeat/module/prometheus/stats/stats.go new file mode 100644 index 000000000000..1429f6c76ba2 --- /dev/null +++ b/metricbeat/module/prometheus/stats/stats.go @@ -0,0 +1,84 @@ +package stats + +import ( + "bufio" + "fmt" + "net/http" + "strings" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/metrics" +) + +var ( + debugf = logp.MakeDebug("prometheus-stats") + + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +func init() { + if err := mb.Registry.AddMetricSet("prometheus", "stats", New, hostParser); err != nil { + panic(err) + } +} + +type MetricSet struct { + mb.BaseMetricSet + client *http.Client +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("EXPERIMENTAL: The prometheus stats metricset is experimental") + + return &MetricSet{ + BaseMetricSet: base, + client: &http.Client{Timeout: base.Module().Config().Timeout}, + }, nil +} + +func (m *MetricSet) Fetch() (common.MapStr, error) { + + req, err := http.NewRequest("GET", m.HostData().SanitizedURI, nil) + if m.HostData().User != "" || m.HostData().Password != "" { + req.SetBasicAuth(m.HostData().User, m.HostData().Password) + } + resp, err := m.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making http request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) + } + + scanner := bufio.NewScanner(resp.Body) + + entries := map[string]interface{}{} + + // Iterate through all events to gather data + for scanner.Scan() { + line := scanner.Text() + + // Skip comments and calculated lines + if line[0] == '#' || strings.Contains(line, "quantile=") { + continue + } + split := strings.Split(line, " ") + entries[split[0]] = split[1] + } + + data, err := eventMapping(entries) + + return data, err +} diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go new file mode 100644 index 000000000000..5c1f70a5d20c --- /dev/null +++ b/metricbeat/module/prometheus/stats/stats_integration_test.go @@ -0,0 +1,62 @@ +// +build integration + +package stats + +import ( + "os" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + + // Check number of fields. + assert.Equal(t, 3, len(event)) + assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"stats"}, + "hosts": []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()}, + } +} + +func getPrometheusEnvHost() string { + host := os.Getenv("PROMETHEUS_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func getPrometheusEnvPort() string { + port := os.Getenv("PROMETHEUS_PORT") + + if len(port) == 0 { + port = "9090" + } + return port +} diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py new file mode 100644 index 000000000000..b9b0fbcfa2e3 --- /dev/null +++ b/metricbeat/tests/system/test_prometheus.py @@ -0,0 +1,38 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + +PROMETHEUS_FIELDS = metricbeat.COMMON_FIELDS + ["prometheus"] + +class Test(metricbeat.BaseTest): + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_stats(self): + """ + prometheus stats test + """ + self.render_config_template(modules=[{ + "name": "prometheus", + "metricsets": ["stats"], + "hosts": self.get_hosts(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log.replace("WARN EXPERIMENTAL", ""), "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return ["http://" + os.getenv('PROMETHEUS_HOST', 'localhost') + ':' + + os.getenv('PROMETHEUS_PORT', '9090')]