From eb093ffb9e9af082f01a312e7b254ef8e19223d4 Mon Sep 17 00:00:00 2001 From: Alex Paskulin Date: Tue, 12 Feb 2019 09:16:52 -0800 Subject: [PATCH] Copy Prometheus guide to all versions (#1190) * re-org, add asset * remove asset due to filter syntax issue see https://github.com/sensu/sensu-prometheus-collector/pull/14 * landing page fixups * articles * fixups * copy to all versions --- .../5.0/getting-started/prometheus-metrics.md | 215 +++++++----- .../sensu-go/5.0/getting-started/sandbox.md | 5 +- .../5.1/files/up_or_down_dashboard.json | 207 +++++++++++ .../5.1/getting-started/prometheus-metrics.md | 332 ++++++++++++++++++ .../sensu-go/5.1/getting-started/sandbox.md | 3 + .../5.2/files/up_or_down_dashboard.json | 207 +++++++++++ .../5.2/getting-started/prometheus-metrics.md | 332 ++++++++++++++++++ .../sensu-go/5.2/getting-started/sandbox.md | 3 + 8 files changed, 1219 insertions(+), 85 deletions(-) create mode 100644 content/sensu-go/5.1/files/up_or_down_dashboard.json create mode 100644 content/sensu-go/5.1/getting-started/prometheus-metrics.md create mode 100644 content/sensu-go/5.2/files/up_or_down_dashboard.json create mode 100644 content/sensu-go/5.2/getting-started/prometheus-metrics.md diff --git a/content/sensu-go/5.0/getting-started/prometheus-metrics.md b/content/sensu-go/5.0/getting-started/prometheus-metrics.md index d8812a316f..dc1dd249ca 100644 --- a/content/sensu-go/5.0/getting-started/prometheus-metrics.md +++ b/content/sensu-go/5.0/getting-started/prometheus-metrics.md @@ -6,14 +6,20 @@ product: "Sensu Go" platformContent: false --- -- [What is the Sensu Prometheus Collector?](#what-is-the-sensu-prometheus-collector) -- [Why use Sensu with Prometheus?](#why-use-sensu-with-prometheus) -- [Install and configure Prometheus](#install-and-configure-prometheus) -- [Install and configure Sensu Go](#install-and-configure-sensu-go) -- [Install Sensu Prometheus Collector](#install-sensu-prometheus-collector) -- [Install Sensu InfluxDB Handler](#install-sensu-influxdb-handler) -- [Install and configure InfluxDB](#install-and-configure-influxdb) -- [Install and configure Grafana](#install-and-configure-grafana) +- [Set up](#set-up) + - [Install and configure Prometheus](#install-and-configure-prometheus) + - [Install and configure Sensu Go](#install-and-configure-sensu-go) + - [Install and configure InfluxDB](#install-and-configure-influxdb) + - [Install and configure Grafana](#install-and-configure-grafana) +- [Create a Sensu InfluxDB pipeline](#create-a-sensu-influxdb-pipeline) + - [Install Sensu InfluxDB handler](#install-sensu-influxdb-handler) + - [Create a Sensu handler](#create-a-sensu-handler) +- [Collect Prometheus metrics with Sensu](#collect-prometheus-metrics-with-sensu) + - [Install Sensu Prometheus Collector](#install-sensu-prometheus-collector) + - [Add a Sensu check to complete the pipeline](#add-a-sensu-check-to-complete-the-pipeline) +- [Visualize metrics with Grafana](#visualize-metrics-with-grafana) + - [Configure a dashboard in Grafana](#configure-a-dashboard-in-grafana) + - [View metrics in Grafana](#view-metrics-in-grafana) ## What is the Sensu Prometheus Collector? @@ -29,7 +35,9 @@ This guide uses CentOS 7 as the operating system with all components running on At the end, you will have Prometheus scraping metrics. The Sensu Prometheus Collector will then query the Prometheus API as a Sensu check, send those to an InfluxDB Sensu handler, which will send metrics to an InfluxDB instance. Finally, Grafana will query InfluxDB to display those collected metrics. -### Install and Configure Prometheus +## Set up + +### Install and configure Prometheus Download and extract Prometheus. @@ -94,32 +102,6 @@ systemctl status sensu-backend systemctl status sensu-agent {{< /highlight >}} -### Install Sensu Prometheus Collector - -{{< highlight shell >}} -wget -q -nc https://github.com/sensu/sensu-prometheus-collector/releases/download/1.1.4/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -P /tmp/ - -tar xvfz /tmp/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -C /tmp/ - -cp /tmp/bin/sensu-prometheus-collector /usr/local/bin/ -{{< /highlight >}} - -Confirm the collector can get metrics from Prometheus. - -{{< highlight shell >}} -/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up -{{< /highlight >}} - -### Install Sensu InfluxDB handler - -{{< highlight shell >}} -wget -q -nc https://github.com/sensu/sensu-influxdb-handler/releases/download/3.0.1/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -P /tmp/ - -tar xvfz /tmp/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -C /tmp/ - -cp /tmp/bin/sensu-influxdb-handler /usr/local/bin/ -{{< /highlight >}} - ### Install and configure InfluxDB Add InfluxDB repo. @@ -163,33 +145,63 @@ influx -execute "CREATE USER sensu WITH PASSWORD 'sensu'" influx -execute "GRANT ALL ON sensu TO sensu" {{< /highlight >}} -### Add Sensu check and handler to complete the pipeline +### Install and configure Grafana -Given the following resource definition in a file called `resources.json`: +Install Grafana. + +{{< highlight shell >}} +sudo yum install -y https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm +{{< /highlight >}} + +Change Grafana's listen port to not conflict with the Sensu Dashboard. + +{{< highlight shell >}} +sudo sed -i 's/^;http_port = 3000/http_port = 4000/' /etc/grafana/grafana.ini +{{< /highlight >}} + +Create a `/etc/grafana/provisioning/datasources/influxdb.yaml` file, and add an InfluxDB data source. + +{{< highlight yml >}} +apiVersion: 1 + +deleteDatasources: + - name: InfluxDB + orgId: 1 + +datasources: + - name: InfluxDB + type: influxdb + access: proxy + orgId: 1 + database: sensu + user: grafana + password: grafana + url: http://localhost:8086 +{{< /highlight >}} + +Start Grafana. + +{{< highlight shell >}} +systemctl start grafana-server +{{< /highlight >}} + +## Create a Sensu InfluxDB pipeline + +### Install Sensu InfluxDB handler + +{{< highlight shell >}} +wget -q -nc https://github.com/sensu/sensu-influxdb-handler/releases/download/3.0.1/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -P /tmp/ + +tar xvfz /tmp/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -C /tmp/ + +cp /tmp/bin/sensu-influxdb-handler /usr/local/bin/ +{{< /highlight >}} + +### Create a Sensu handler + +Given the following handler definition in a file called `handler.json`: {{< highlight shell >}} -{ - "type": "CheckConfig", - "api_version": "core/v2", - "metadata": { - "name": "prometheus_metrics", - "namespace": "default" - }, - "spec": { - "command": "/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up", - "handlers": [ - "influxdb" - ], - "interval": 10, - "publish": true, - "output_metric_format": "influxdb_line", - "output_metric_handlers": [], - "subscriptions": [ - "app_tier" - ], - "timeout": 0 - } -} { "type": "Handler", "api_version": "core/v2", @@ -206,64 +218,99 @@ Given the following resource definition in a file called `resources.json`: } {{< /highlight >}} -Use `sensuctl` to add the check and handler to the Sensu backend. +Use `sensuctl` to add the handler to Sensu. {{< highlight shell >}} -sensuctl create -f resources.json +sensuctl create --file handler.json {{< /highlight >}} -### Install and configure Grafana +## Collect Prometheus metrics with Sensu -Install Grafana. +### Install Sensu Prometheus Collector {{< highlight shell >}} -sudo yum install -y https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm +wget -q -nc https://github.com/sensu/sensu-prometheus-collector/releases/download/1.1.4/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -P /tmp/ + +tar xvfz /tmp/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -C /tmp/ + +cp /tmp/bin/sensu-prometheus-collector /usr/local/bin/ {{< /highlight >}} -Change Grafana's listen port to not conflict with the Sensu Dashboard. +Confirm the collector can get metrics from Prometheus. {{< highlight shell >}} -sudo sed -i 's/^;http_port = 3000/http_port = 4000/' /etc/grafana/grafana.ini +/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up +up,instance=localhost:9090,job=prometheus value=1 1549991087 {{< /highlight >}} -Add an InfluxDB data source in `/etc/grafana/provisioning/datasources/influxdb.yaml`. +### Add a Sensu check to complete the pipeline -{{< highlight yml >}} -apiVersion: 1 +Given the following check definition in a file called `check.json`: -deleteDatasources: - - name: InfluxDB - orgId: 1 +{{< highlight shell >}} +{ + "type": "CheckConfig", + "api_version": "core/v2", + "metadata": { + "name": "prometheus_metrics", + "namespace": "default" + }, + "spec": { + "command": "/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up", + "handlers": [ + "influxdb" + ], + "interval": 10, + "publish": true, + "output_metric_format": "influxdb_line", + "output_metric_handlers": [], + "subscriptions": [ + "app_tier" + ], + "timeout": 0 + } +} +{{< /highlight >}} -datasources: - - name: InfluxDB - type: influxdb - access: proxy - orgId: 1 - database: sensu - user: grafana - password: grafana - url: http://localhost:8086 +_PRO TIP: `sensuctl create -f` also accepts files containing multiple resources definitions._ + +Use `sensuctl` to add the check to Sensu. + +{{< highlight shell >}} +sensuctl create --file check.json {{< /highlight >}} -Start Grafana. +We can see the events generated by the `prometheus_metrics` check in the Sensu dashboard. +Visit http://127.0.0.1:3000, and log in as the default admin user: username `admin` and password `P@ssw0rd!`. + +We can also see the metric event data using sensuctl. {{< highlight shell >}} -systemctl start grafana-server +sensuctl event list + Entity Check Output Status Silenced Timestamp +────────────── ──────────────────── ──────────────────────────────────────────────────────────────────────── ──────── ────────── ─────────────────────────────── +sensu-centos keepalive Keepalive last sent from sensu-centos at 2019-02-12 01:01:37 +0000 UTC 0 false 2019-02-12 01:01:37 +0000 UTC +sensu-centos prometheus_metrics up,instance=localhost:9090,job=prometheus value=1 1549933306 0 false 2019-02-12 01:01:46 +0000 UTC {{< /highlight >}} +## Visualize metrics with Grafana + +### Configure a dashboard in Grafana + Download the Grafana dashboard configuration file from the Sensu docs. {{< highlight shell >}} wget https://docs.sensu.io/sensu-go/5.0/files/up_or_down_dashboard.json {{< /highlight >}} -Using the just created file, add the dashboard to Grafana using an API call. +Using the downloaded file, add the dashboard to Grafana using an API call. {{< highlight shell >}} curl -s -XPOST -H 'Content-Type: application/json' -d@up_or_down_dashboard.json HTTP://admin:admin@127.0.0.1:4000/api/dashboards/db {{< /highlight >}} +### View metrics in Grafana + Confirm metrics in Grafana with `admin:admin` login at http://127.0.0.1:4000. Once logged in, click on Home in the upper left corner, then below click on the Up or Down Sample 2 dashboard. Once there, you should see a graph that has started showing metrics like this diff --git a/content/sensu-go/5.0/getting-started/sandbox.md b/content/sensu-go/5.0/getting-started/sandbox.md index 0f515a4902..63292a1233 100644 --- a/content/sensu-go/5.0/getting-started/sandbox.md +++ b/content/sensu-go/5.0/getting-started/sandbox.md @@ -16,7 +16,10 @@ Welcome to the Sensu sandbox! The sandbox is the best place to get started with - [Start here](../learn-sensu): Building your first monitoring workflow ### Container monitoring -- [Container and application monitoring with Sensu](../sample-app): Monitoring a sample app on Kubernetes +- [Container and application monitoring with Sensu](../sample-app): Monitoring a Kubernetes sample app + +### Metrics +- [Sensu + Prometheus](../prometheus-metrics): Collecting Prometheus metrics with Sensu ### Upgrading from Sensu 1.x to Sensu Go - [Sensu translator](https://github.com/sensu/sandbox/tree/master/sensu-go/lesson_plans/check-upgrade): Translating check configuration diff --git a/content/sensu-go/5.1/files/up_or_down_dashboard.json b/content/sensu-go/5.1/files/up_or_down_dashboard.json new file mode 100644 index 0000000000..ecd972353a --- /dev/null +++ b/content/sensu-go/5.1/files/up_or_down_dashboard.json @@ -0,0 +1,207 @@ +{ + "dashboard": { + "__inputs": [ + { + "name": "InfluxDB", + "label": "InfluxDB", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.1.4" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "up", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Up or Down", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Up or Down Sample 2", + "uid": "nb442Vsiz", + "version": 1 + } +} diff --git a/content/sensu-go/5.1/getting-started/prometheus-metrics.md b/content/sensu-go/5.1/getting-started/prometheus-metrics.md new file mode 100644 index 0000000000..75354a1674 --- /dev/null +++ b/content/sensu-go/5.1/getting-started/prometheus-metrics.md @@ -0,0 +1,332 @@ +--- +title: "Using the Sensu Prometheus Collector" +version: "5.1" +description: "The Sensu Prometheus Collector is a check plugin that collects metrics from a Prometheus exporter or the Prometheus query API. This allows Sensu to route the collected metrics to a time series database, like InfluxDB. Follow the guide to get started collecting Prometheus metrics with Sensu." +product: "Sensu Go" +platformContent: false +--- + +- [Set up](#set-up) + - [Install and configure Prometheus](#install-and-configure-prometheus) + - [Install and configure Sensu Go](#install-and-configure-sensu-go) + - [Install and configure InfluxDB](#install-and-configure-influxdb) + - [Install and configure Grafana](#install-and-configure-grafana) +- [Create a Sensu InfluxDB pipeline](#create-a-sensu-influxdb-pipeline) + - [Install Sensu InfluxDB handler](#install-sensu-influxdb-handler) + - [Create a Sensu handler](#create-a-sensu-handler) +- [Collect Prometheus metrics with Sensu](#collect-prometheus-metrics-with-sensu) + - [Install Sensu Prometheus Collector](#install-sensu-prometheus-collector) + - [Add a Sensu check to complete the pipeline](#add-a-sensu-check-to-complete-the-pipeline) +- [Visualize metrics with Grafana](#visualize-metrics-with-grafana) + - [Configure a dashboard in Grafana](#configure-a-dashboard-in-grafana) + - [View metrics in Grafana](#view-metrics-in-grafana) + +## What is the Sensu Prometheus Collector? + +The [Sensu Prometheus Collector][1] is a check plugin that collects metrics from a [Prometheus exporter][2] or the [Prometheus query API][3]. This allows Sensu to route the collected metrics to one or more time series databases, such as InfluxDB or Graphite. + +## Why use Sensu with Prometheus? + +The Prometheus ecosystem contains a number of actively maintained exporters, such as the [node exporter][5] for reporting hardware and operating system metrics or Google's [cAdvisor exporter][6] for monitoring containers. These exporters expose metrics which Sensu can collect and route to one or more time series databases, such as InfluxDB or Graphite. Both Sensu and Prometheus can run in parallel, complimenting each other and making use of environments where Prometheus is already deployed. + +## In this guide + +This guide uses CentOS 7 as the operating system with all components running on the same compute resource. Commands and steps may change for different distributions or if components are running on different compute resources. + +At the end, you will have Prometheus scraping metrics. The Sensu Prometheus Collector will then query the Prometheus API as a Sensu check, send those to an InfluxDB Sensu handler, which will send metrics to an InfluxDB instance. Finally, Grafana will query InfluxDB to display those collected metrics. + +## Set up + +### Install and configure Prometheus + +Download and extract Prometheus. + +{{< highlight shell >}} +wget https://github.com/prometheus/prometheus/releases/download/v2.6.0/prometheus-2.6.0.linux-amd64.tar.gz + +tar xvfz prometheus-*.tar.gz + +cd prometheus-* +{{< /highlight >}} + +Replace the default `prometheus.yml` configuration file with the following configuration. + +{{< highlight shell >}} +global: + scrape_interval: 15s + external_labels: + monitor: 'codelab-monitor' + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] +{{< /highlight >}} + +Start Prometheus in the background. + +{{< highlight shell >}} +nohup ./prometheus --config.file=prometheus.yml > prometheus.log 2>&1 & +[1] 7647 +{{< /highlight >}} + +Ensure Prometheus is running. + +{{< highlight shell >}} +ps -ef | grep prometheus +vagrant 7647 3937 2 22:23 pts/0 00:00:00 ./prometheus --config.file=prometheus.yml +{{< /highlight >}} + +### Install and configure Sensu Go + +Follow the RHEL/CentOS [install instructions][4] for the Sensu backend, Sensu agent and sensuctl. + +Add an `app_tier` subscription to `/etc/sensu/agent.yml`. + +{{< highlight shell >}} +subscriptions: + - "app_tier" +{{< /highlight >}} + +Restart the sensu agent to apply the configuration change. + +{{< highlight shell >}} +systemctl restart sensu-agent +{{< /highlight >}} + +Ensure Sensu services are running. + +{{< highlight shell >}} +systemctl status sensu-backend +systemctl status sensu-agent +{{< /highlight >}} + +### Install and configure InfluxDB + +Add InfluxDB repo. + +{{< highlight shell >}} +echo "[influxdb] +name = InfluxDB Repository - RHEL \$releasever +baseurl = https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable +enabled = 1 +gpgcheck = 1 +gpgkey = https://repos.influxdata.com/influxdb.key" | sudo tee /etc/yum.repos.d/influxdb.repo +{{< /highlight >}} + +Install InfluxDB. + +{{< highlight shell >}} +sudo yum -y install influxdb +{{< /highlight >}} + +Open `/etc/influxdb/influxdb.conf` and uncomment the `http` API line. + +{{< highlight shell >}} +[http] + # Determines whether HTTP endpoint is enabled. + enabled = true +{{< /highlight >}} + +Start InfluxDB. + +{{< highlight shell >}} +sudo systemctl start influxdb +{{< /highlight >}} + +Add the Sensu user and database. + +{{< highlight shell >}} +influx -execute "CREATE DATABASE sensu" + +influx -execute "CREATE USER sensu WITH PASSWORD 'sensu'" + +influx -execute "GRANT ALL ON sensu TO sensu" +{{< /highlight >}} + +### Install and configure Grafana + +Install Grafana. + +{{< highlight shell >}} +sudo yum install -y https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm +{{< /highlight >}} + +Change Grafana's listen port to not conflict with the Sensu Dashboard. + +{{< highlight shell >}} +sudo sed -i 's/^;http_port = 3000/http_port = 4000/' /etc/grafana/grafana.ini +{{< /highlight >}} + +Create a `/etc/grafana/provisioning/datasources/influxdb.yaml` file, and add an InfluxDB data source. + +{{< highlight yml >}} +apiVersion: 1 + +deleteDatasources: + - name: InfluxDB + orgId: 1 + +datasources: + - name: InfluxDB + type: influxdb + access: proxy + orgId: 1 + database: sensu + user: grafana + password: grafana + url: http://localhost:8086 +{{< /highlight >}} + +Start Grafana. + +{{< highlight shell >}} +systemctl start grafana-server +{{< /highlight >}} + +## Create a Sensu InfluxDB pipeline + +### Install Sensu InfluxDB handler + +{{< highlight shell >}} +wget -q -nc https://github.com/sensu/sensu-influxdb-handler/releases/download/3.0.1/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -P /tmp/ + +tar xvfz /tmp/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -C /tmp/ + +cp /tmp/bin/sensu-influxdb-handler /usr/local/bin/ +{{< /highlight >}} + +### Create a Sensu handler + +Given the following handler definition in a file called `handler.json`: + +{{< highlight shell >}} +{ + "type": "Handler", + "api_version": "core/v2", + "metadata": { + "name": "influxdb", + "namespace": "default" + }, + "spec": { + "command": "/usr/local/bin/sensu-influxdb-handler -a 'http://127.0.0.1:8086' -d sensu -u sensu -p sensu", + "env_vars": [], + "timeout": 10, + "type": "pipe" + } +} +{{< /highlight >}} + +Use `sensuctl` to add the handler to Sensu. + +{{< highlight shell >}} +sensuctl create --file handler.json +{{< /highlight >}} + +## Collect Prometheus metrics with Sensu + +### Install Sensu Prometheus Collector + +{{< highlight shell >}} +wget -q -nc https://github.com/sensu/sensu-prometheus-collector/releases/download/1.1.4/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -P /tmp/ + +tar xvfz /tmp/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -C /tmp/ + +cp /tmp/bin/sensu-prometheus-collector /usr/local/bin/ +{{< /highlight >}} + +Confirm the collector can get metrics from Prometheus. + +{{< highlight shell >}} +/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up +up,instance=localhost:9090,job=prometheus value=1 1549991087 +{{< /highlight >}} + +### Add a Sensu check to complete the pipeline + +Given the following check definition in a file called `check.json`: + +{{< highlight shell >}} +{ + "type": "CheckConfig", + "api_version": "core/v2", + "metadata": { + "name": "prometheus_metrics", + "namespace": "default" + }, + "spec": { + "command": "/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up", + "handlers": [ + "influxdb" + ], + "interval": 10, + "publish": true, + "output_metric_format": "influxdb_line", + "output_metric_handlers": [], + "subscriptions": [ + "app_tier" + ], + "timeout": 0 + } +} +{{< /highlight >}} + +_PRO TIP: `sensuctl create -f` also accepts files containing multiple resources definitions._ + +Use `sensuctl` to add the check to Sensu. + +{{< highlight shell >}} +sensuctl create --file check.json +{{< /highlight >}} + +We can see the events generated by the `prometheus_metrics` check in the Sensu dashboard. +Visit http://127.0.0.1:3000, and log in as the default admin user: username `admin` and password `P@ssw0rd!`. + +We can also see the metric event data using sensuctl. + +{{< highlight shell >}} +sensuctl event list + Entity Check Output Status Silenced Timestamp +────────────── ──────────────────── ──────────────────────────────────────────────────────────────────────── ──────── ────────── ─────────────────────────────── +sensu-centos keepalive Keepalive last sent from sensu-centos at 2019-02-12 01:01:37 +0000 UTC 0 false 2019-02-12 01:01:37 +0000 UTC +sensu-centos prometheus_metrics up,instance=localhost:9090,job=prometheus value=1 1549933306 0 false 2019-02-12 01:01:46 +0000 UTC +{{< /highlight >}} + +## Visualize metrics with Grafana + +### Configure a dashboard in Grafana + +Download the Grafana dashboard configuration file from the Sensu docs. + +{{< highlight shell >}} +wget https://docs.sensu.io/sensu-go/5.1/files/up_or_down_dashboard.json +{{< /highlight >}} + +Using the downloaded file, add the dashboard to Grafana using an API call. + +{{< highlight shell >}} +curl -s -XPOST -H 'Content-Type: application/json' -d@up_or_down_dashboard.json HTTP://admin:admin@127.0.0.1:4000/api/dashboards/db +{{< /highlight >}} + +### View metrics in Grafana + +Confirm metrics in Grafana with `admin:admin` login at http://127.0.0.1:4000. + +Once logged in, click on Home in the upper left corner, then below click on the Up or Down Sample 2 dashboard. Once there, you should see a graph that has started showing metrics like this + +![up_or_down_detail](/images/prometheus-collector/up_or_down_detail.png) + +## Conclusion + +You should now have a working setup with Prometheus scraping metrics. The Sensu Prometheus Collecting is being ran via a Sensu check and collecting those metrics from Prometheus' API. The metrics are then handled by the InfluxDB handler, sent to InfluxDB and then visualized by a Grafana Dashboard. + +Using this information, you can now plug the Sensu Prometheus Collector into your Sensu ecosystem and leverage Prometheus to gather metrics and Sensu to send them to the proper final destination. Prometheus has a [comprehensive list][7] of additional exporters to pull in metrics. + +[1]: https://github.com/sensu/sensu-prometheus-collector +[2]: https://prometheus.io/docs/instrumenting/exporters/ +[3]: https://prometheus.io/docs/prometheus/latest/querying/api/ +[4]: ../../installation/install-sensu/ +[5]: https://github.com/prometheus/node_exporter +[6]: https://github.com/google/cadvisor +[7]: https://prometheus.io/docs/instrumenting/exporters/ diff --git a/content/sensu-go/5.1/getting-started/sandbox.md b/content/sensu-go/5.1/getting-started/sandbox.md index 7de4499bdb..057ea76cbd 100644 --- a/content/sensu-go/5.1/getting-started/sandbox.md +++ b/content/sensu-go/5.1/getting-started/sandbox.md @@ -18,5 +18,8 @@ Welcome to the Sensu sandbox! The sandbox is the best place to get started with ### Container monitoring - [Container and application monitoring with Sensu](../sample-app): Monitoring a sample app on Kubernetes +### Metrics +- [Sensu + Prometheus](../prometheus-metrics): Collecting Prometheus metrics with Sensu + ### Upgrading from Sensu 1.x to Sensu Go - [Sensu translator](https://github.com/sensu/sandbox/tree/master/sensu-go/lesson_plans/check-upgrade): Translating check configuration diff --git a/content/sensu-go/5.2/files/up_or_down_dashboard.json b/content/sensu-go/5.2/files/up_or_down_dashboard.json new file mode 100644 index 0000000000..ecd972353a --- /dev/null +++ b/content/sensu-go/5.2/files/up_or_down_dashboard.json @@ -0,0 +1,207 @@ +{ + "dashboard": { + "__inputs": [ + { + "name": "InfluxDB", + "label": "InfluxDB", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.1.4" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "up", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Up or Down", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Up or Down Sample 2", + "uid": "nb442Vsiz", + "version": 1 + } +} diff --git a/content/sensu-go/5.2/getting-started/prometheus-metrics.md b/content/sensu-go/5.2/getting-started/prometheus-metrics.md new file mode 100644 index 0000000000..86cc0eb481 --- /dev/null +++ b/content/sensu-go/5.2/getting-started/prometheus-metrics.md @@ -0,0 +1,332 @@ +--- +title: "Using the Sensu Prometheus Collector" +version: "5.2" +description: "The Sensu Prometheus Collector is a check plugin that collects metrics from a Prometheus exporter or the Prometheus query API. This allows Sensu to route the collected metrics to a time series database, like InfluxDB. Follow the guide to get started collecting Prometheus metrics with Sensu." +product: "Sensu Go" +platformContent: false +--- + +- [Set up](#set-up) + - [Install and configure Prometheus](#install-and-configure-prometheus) + - [Install and configure Sensu Go](#install-and-configure-sensu-go) + - [Install and configure InfluxDB](#install-and-configure-influxdb) + - [Install and configure Grafana](#install-and-configure-grafana) +- [Create a Sensu InfluxDB pipeline](#create-a-sensu-influxdb-pipeline) + - [Install Sensu InfluxDB handler](#install-sensu-influxdb-handler) + - [Create a Sensu handler](#create-a-sensu-handler) +- [Collect Prometheus metrics with Sensu](#collect-prometheus-metrics-with-sensu) + - [Install Sensu Prometheus Collector](#install-sensu-prometheus-collector) + - [Add a Sensu check to complete the pipeline](#add-a-sensu-check-to-complete-the-pipeline) +- [Visualize metrics with Grafana](#visualize-metrics-with-grafana) + - [Configure a dashboard in Grafana](#configure-a-dashboard-in-grafana) + - [View metrics in Grafana](#view-metrics-in-grafana) + +## What is the Sensu Prometheus Collector? + +The [Sensu Prometheus Collector][1] is a check plugin that collects metrics from a [Prometheus exporter][2] or the [Prometheus query API][3]. This allows Sensu to route the collected metrics to one or more time series databases, such as InfluxDB or Graphite. + +## Why use Sensu with Prometheus? + +The Prometheus ecosystem contains a number of actively maintained exporters, such as the [node exporter][5] for reporting hardware and operating system metrics or Google's [cAdvisor exporter][6] for monitoring containers. These exporters expose metrics which Sensu can collect and route to one or more time series databases, such as InfluxDB or Graphite. Both Sensu and Prometheus can run in parallel, complimenting each other and making use of environments where Prometheus is already deployed. + +## In this guide + +This guide uses CentOS 7 as the operating system with all components running on the same compute resource. Commands and steps may change for different distributions or if components are running on different compute resources. + +At the end, you will have Prometheus scraping metrics. The Sensu Prometheus Collector will then query the Prometheus API as a Sensu check, send those to an InfluxDB Sensu handler, which will send metrics to an InfluxDB instance. Finally, Grafana will query InfluxDB to display those collected metrics. + +## Set up + +### Install and configure Prometheus + +Download and extract Prometheus. + +{{< highlight shell >}} +wget https://github.com/prometheus/prometheus/releases/download/v2.6.0/prometheus-2.6.0.linux-amd64.tar.gz + +tar xvfz prometheus-*.tar.gz + +cd prometheus-* +{{< /highlight >}} + +Replace the default `prometheus.yml` configuration file with the following configuration. + +{{< highlight shell >}} +global: + scrape_interval: 15s + external_labels: + monitor: 'codelab-monitor' + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] +{{< /highlight >}} + +Start Prometheus in the background. + +{{< highlight shell >}} +nohup ./prometheus --config.file=prometheus.yml > prometheus.log 2>&1 & +[1] 7647 +{{< /highlight >}} + +Ensure Prometheus is running. + +{{< highlight shell >}} +ps -ef | grep prometheus +vagrant 7647 3937 2 22:23 pts/0 00:00:00 ./prometheus --config.file=prometheus.yml +{{< /highlight >}} + +### Install and configure Sensu Go + +Follow the RHEL/CentOS [install instructions][4] for the Sensu backend, Sensu agent and sensuctl. + +Add an `app_tier` subscription to `/etc/sensu/agent.yml`. + +{{< highlight shell >}} +subscriptions: + - "app_tier" +{{< /highlight >}} + +Restart the sensu agent to apply the configuration change. + +{{< highlight shell >}} +systemctl restart sensu-agent +{{< /highlight >}} + +Ensure Sensu services are running. + +{{< highlight shell >}} +systemctl status sensu-backend +systemctl status sensu-agent +{{< /highlight >}} + +### Install and configure InfluxDB + +Add InfluxDB repo. + +{{< highlight shell >}} +echo "[influxdb] +name = InfluxDB Repository - RHEL \$releasever +baseurl = https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable +enabled = 1 +gpgcheck = 1 +gpgkey = https://repos.influxdata.com/influxdb.key" | sudo tee /etc/yum.repos.d/influxdb.repo +{{< /highlight >}} + +Install InfluxDB. + +{{< highlight shell >}} +sudo yum -y install influxdb +{{< /highlight >}} + +Open `/etc/influxdb/influxdb.conf` and uncomment the `http` API line. + +{{< highlight shell >}} +[http] + # Determines whether HTTP endpoint is enabled. + enabled = true +{{< /highlight >}} + +Start InfluxDB. + +{{< highlight shell >}} +sudo systemctl start influxdb +{{< /highlight >}} + +Add the Sensu user and database. + +{{< highlight shell >}} +influx -execute "CREATE DATABASE sensu" + +influx -execute "CREATE USER sensu WITH PASSWORD 'sensu'" + +influx -execute "GRANT ALL ON sensu TO sensu" +{{< /highlight >}} + +### Install and configure Grafana + +Install Grafana. + +{{< highlight shell >}} +sudo yum install -y https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm +{{< /highlight >}} + +Change Grafana's listen port to not conflict with the Sensu Dashboard. + +{{< highlight shell >}} +sudo sed -i 's/^;http_port = 3000/http_port = 4000/' /etc/grafana/grafana.ini +{{< /highlight >}} + +Create a `/etc/grafana/provisioning/datasources/influxdb.yaml` file, and add an InfluxDB data source. + +{{< highlight yml >}} +apiVersion: 1 + +deleteDatasources: + - name: InfluxDB + orgId: 1 + +datasources: + - name: InfluxDB + type: influxdb + access: proxy + orgId: 1 + database: sensu + user: grafana + password: grafana + url: http://localhost:8086 +{{< /highlight >}} + +Start Grafana. + +{{< highlight shell >}} +systemctl start grafana-server +{{< /highlight >}} + +## Create a Sensu InfluxDB pipeline + +### Install Sensu InfluxDB handler + +{{< highlight shell >}} +wget -q -nc https://github.com/sensu/sensu-influxdb-handler/releases/download/3.0.1/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -P /tmp/ + +tar xvfz /tmp/sensu-influxdb-handler_3.0.1_linux_amd64.tar.gz -C /tmp/ + +cp /tmp/bin/sensu-influxdb-handler /usr/local/bin/ +{{< /highlight >}} + +### Create a Sensu handler + +Given the following handler definition in a file called `handler.json`: + +{{< highlight shell >}} +{ + "type": "Handler", + "api_version": "core/v2", + "metadata": { + "name": "influxdb", + "namespace": "default" + }, + "spec": { + "command": "/usr/local/bin/sensu-influxdb-handler -a 'http://127.0.0.1:8086' -d sensu -u sensu -p sensu", + "env_vars": [], + "timeout": 10, + "type": "pipe" + } +} +{{< /highlight >}} + +Use `sensuctl` to add the handler to Sensu. + +{{< highlight shell >}} +sensuctl create --file handler.json +{{< /highlight >}} + +## Collect Prometheus metrics with Sensu + +### Install Sensu Prometheus Collector + +{{< highlight shell >}} +wget -q -nc https://github.com/sensu/sensu-prometheus-collector/releases/download/1.1.4/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -P /tmp/ + +tar xvfz /tmp/sensu-prometheus-collector_1.1.4_linux_386.tar.gz -C /tmp/ + +cp /tmp/bin/sensu-prometheus-collector /usr/local/bin/ +{{< /highlight >}} + +Confirm the collector can get metrics from Prometheus. + +{{< highlight shell >}} +/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up +up,instance=localhost:9090,job=prometheus value=1 1549991087 +{{< /highlight >}} + +### Add a Sensu check to complete the pipeline + +Given the following check definition in a file called `check.json`: + +{{< highlight shell >}} +{ + "type": "CheckConfig", + "api_version": "core/v2", + "metadata": { + "name": "prometheus_metrics", + "namespace": "default" + }, + "spec": { + "command": "/usr/local/bin/sensu-prometheus-collector -prom-url http://localhost:9090 -prom-query up", + "handlers": [ + "influxdb" + ], + "interval": 10, + "publish": true, + "output_metric_format": "influxdb_line", + "output_metric_handlers": [], + "subscriptions": [ + "app_tier" + ], + "timeout": 0 + } +} +{{< /highlight >}} + +_PRO TIP: `sensuctl create -f` also accepts files containing multiple resources definitions._ + +Use `sensuctl` to add the check to Sensu. + +{{< highlight shell >}} +sensuctl create --file check.json +{{< /highlight >}} + +We can see the events generated by the `prometheus_metrics` check in the Sensu dashboard. +Visit http://127.0.0.1:3000, and log in as the default admin user: username `admin` and password `P@ssw0rd!`. + +We can also see the metric event data using sensuctl. + +{{< highlight shell >}} +sensuctl event list + Entity Check Output Status Silenced Timestamp +────────────── ──────────────────── ──────────────────────────────────────────────────────────────────────── ──────── ────────── ─────────────────────────────── +sensu-centos keepalive Keepalive last sent from sensu-centos at 2019-02-12 01:01:37 +0000 UTC 0 false 2019-02-12 01:01:37 +0000 UTC +sensu-centos prometheus_metrics up,instance=localhost:9090,job=prometheus value=1 1549933306 0 false 2019-02-12 01:01:46 +0000 UTC +{{< /highlight >}} + +## Visualize metrics with Grafana + +### Configure a dashboard in Grafana + +Download the Grafana dashboard configuration file from the Sensu docs. + +{{< highlight shell >}} +wget https://docs.sensu.io/sensu-go/5.2/files/up_or_down_dashboard.json +{{< /highlight >}} + +Using the downloaded file, add the dashboard to Grafana using an API call. + +{{< highlight shell >}} +curl -s -XPOST -H 'Content-Type: application/json' -d@up_or_down_dashboard.json HTTP://admin:admin@127.0.0.1:4000/api/dashboards/db +{{< /highlight >}} + +### View metrics in Grafana + +Confirm metrics in Grafana with `admin:admin` login at http://127.0.0.1:4000. + +Once logged in, click on Home in the upper left corner, then below click on the Up or Down Sample 2 dashboard. Once there, you should see a graph that has started showing metrics like this + +![up_or_down_detail](/images/prometheus-collector/up_or_down_detail.png) + +## Conclusion + +You should now have a working setup with Prometheus scraping metrics. The Sensu Prometheus Collecting is being ran via a Sensu check and collecting those metrics from Prometheus' API. The metrics are then handled by the InfluxDB handler, sent to InfluxDB and then visualized by a Grafana Dashboard. + +Using this information, you can now plug the Sensu Prometheus Collector into your Sensu ecosystem and leverage Prometheus to gather metrics and Sensu to send them to the proper final destination. Prometheus has a [comprehensive list][7] of additional exporters to pull in metrics. + +[1]: https://github.com/sensu/sensu-prometheus-collector +[2]: https://prometheus.io/docs/instrumenting/exporters/ +[3]: https://prometheus.io/docs/prometheus/latest/querying/api/ +[4]: ../../installation/install-sensu/ +[5]: https://github.com/prometheus/node_exporter +[6]: https://github.com/google/cadvisor +[7]: https://prometheus.io/docs/instrumenting/exporters/ diff --git a/content/sensu-go/5.2/getting-started/sandbox.md b/content/sensu-go/5.2/getting-started/sandbox.md index 38a89e0d72..7ad17455cc 100644 --- a/content/sensu-go/5.2/getting-started/sandbox.md +++ b/content/sensu-go/5.2/getting-started/sandbox.md @@ -18,5 +18,8 @@ Welcome to the Sensu sandbox! The sandbox is the best place to get started with ### Container monitoring - [Container and application monitoring with Sensu](../sample-app): Monitoring a sample app on Kubernetes +### Metrics +- [Sensu + Prometheus](../prometheus-metrics): Collecting Prometheus metrics with Sensu + ### Upgrading from Sensu 1.x to Sensu Go - [Sensu translator](https://github.com/sensu/sandbox/tree/master/sensu-go/lesson_plans/check-upgrade): Translating check configuration