From d50ed18e33ae43e0971b9ac8d6cad9c8034f9df2 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sat, 21 Dec 2019 09:00:53 -0300 Subject: [PATCH] add frontend to the internal prometheus exporter Configures a frontend and a listening port (defaults to 9100) to scrape metrics from an internal Prometheus exporter implementation. Added two new global config keys to configure binding ip and port number. Port as zero removes the frontend. The prometheus exporter is slower and more verbose than the csv exporter, added a note about this in the doc. --- docs/content/en/docs/configuration/keys.md | 42 +++++++++++++------ pkg/converters/ingress/annotations/global.go | 12 +++--- pkg/converters/ingress/annotations/updater.go | 1 - pkg/converters/ingress/defaults.go | 5 +-- pkg/converters/ingress/types/global.go | 2 + pkg/haproxy/instance_test.go | 23 +++++++++- pkg/haproxy/types/types.go | 7 ++++ rootfs/etc/haproxy/template/haproxy.tmpl | 16 +++++++ rootfs/usr/local/etc/haproxy/lua/services.lua | 10 +++++ 9 files changed, 94 insertions(+), 24 deletions(-) diff --git a/docs/content/en/docs/configuration/keys.md b/docs/content/en/docs/configuration/keys.md index 5888f8828..9d8ada39f 100644 --- a/docs/content/en/docs/configuration/keys.md +++ b/docs/content/en/docs/configuration/keys.md @@ -112,10 +112,11 @@ The table below describes all supported configuration keys. | [`bind-fronting-proxy`](#bind) | ip + port | Global | | | [`bind-http`](#bind) | ip + port | Global | | | [`bind-https`](#bind) | ip + port | Global | | -| [`bind-ip-addr-healthz`](#bind-ip-addr) | IP address | Global | `*` | -| [`bind-ip-addr-http`](#bind-ip-addr) | IP address | Global | `*` | -| [`bind-ip-addr-stats`](#bind-ip-addr) | IP address | Global | `*` | -| [`bind-ip-addr-tcp`](#bind-ip-addr) | IP address | Global | `*` | +| [`bind-ip-addr-healthz`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-http`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-prometheus`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-stats`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-tcp`](#bind-ip-addr) | IP address | Global | | | [`blue-green-balance`](#blue-green) | label=value=weight,... | Backend | | | [`blue-green-cookie`](#blue-green) | `CookieName:LabelName` pair | Backend | | | [`blue-green-deploy`](#blue-green) | label=value=weight,... | Backend | | @@ -179,6 +180,7 @@ The table below describes all supported configuration keys. | [`oauth`](#oauth) | "oauth2_proxy" | Backend | | | [`oauth-headers`](#oauth) | `
:,...` | Backend | | | [`oauth-uri-prefix`](#oauth) | URI prefix | Backend | | +| [`prometheus-port`](#bind-port) | port number | Global | `9100` | | [`proxy-body-size`](#proxy-body-size) | size (bytes) | Backend | unlimited | | [`proxy-protocol`](#proxy-protocol) | [v1\|v2\|v2-ssl\|v2-ssl-cn] | Backend | | | [`rewrite-target`](#rewrite-target) | path string | Backend | | @@ -523,24 +525,29 @@ See also: ## Bind IP addr -| Configuration key | Scope | Default | Since | -|------------------------|----------|---------|-------| -| `bind-ip-addr-healthz` | `Global` | `*` | | -| `bind-ip-addr-http` | `Global` | `*` | | -| `bind-ip-addr-stats` | `Global` | `*` | | -| `bind-ip-addr-tcp` | `Global` | `*` | | +| Configuration key | Scope | Default | Since | +|---------------------------|----------|---------|-------| +| `bind-ip-addr-healthz` | `Global` | | | +| `bind-ip-addr-http` | `Global` | | | +| `bind-ip-addr-prometheus` | `Global` | | v0.10 | +| `bind-ip-addr-stats` | `Global` | | | +| `bind-ip-addr-tcp` | `Global` | | | -Define listening IPv4/IPv6 address on public HAProxy frontends. +Define listening IPv4/IPv6 address on public HAProxy frontends. Since v0.10 the default +value changed from `*` to an empty string, which haproxy interprets in the same way and +binds on all IPv4 address. * `bind-ip-addr-tcp`: IP address of all TCP services declared on [`tcp-services`](#tcp-services-configmap) command-line option. -* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared. * `bind-ip-addr-healthz`: IP address of the health check URL. +* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared. +* `bind-ip-addr-prometheus`: IP address of the haproxy's internal Prometheus exporter. * `bind-ip-addr-stats`: IP address of the statistics page. See also [`stats-port`](#stats). See also: * https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#4-bind * [Bind](#bind) +* [Bind port](#bind-port) --- @@ -551,10 +558,21 @@ See also: | `healthz-port` | `Global` | `10253` | | | `http-port` | `Global` | `80` | | | `https-port` | `Global` | `443` | | +| `prometheus-port` | `Global` | `9100` | v0.10 | * `healthz-port`: Define the port number HAProxy should listen to in order to answer for health checking requests. Use `/healthz` as the request path. * `http-port`: Define the port number of unencripted HTTP connections. * `https-port`: Define the port number of encripted HTTPS connections. +* `prometheus-port`: Define the port number of the haproxy's internal Prometheus exporter. This port can be changed to zero `0` to remove the listener. A listener without being scraped does not use system resources, except for the listening port. + +{{% alert title="Note" %}} +The internal Prometheus exporter runs concurrently with request processing, and it is +about 5x slower and 20x more verbose than the CSV exporter. See the haproxy's exporter +[doc](https://github.com/haproxy/haproxy/blob/v2.0.0/contrib/prometheus-exporter/README#L44). +Consider use Prometheus' [haproxy_exporter](https://github.com/prometheus/haproxy_exporter) +on very large clusters - Prometheus' implementation reads the CSV from the stats page and +converts to the Prometheus syntax outside the haproxy process. +{{% /alert %}} See also: diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go index 84dfcf547..05f708ee1 100644 --- a/pkg/converters/ingress/annotations/global.go +++ b/pkg/converters/ingress/annotations/global.go @@ -128,6 +128,13 @@ func (c *updater) buildGlobalProc(d *globalData) { } func (c *updater) buildGlobalStats(d *globalData) { + // healthz + d.global.Healthz.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrHealthz).Value + d.global.Healthz.Port = d.mapper.Get(ingtypes.GlobalHealthzPort).Int() + // prometheus + d.global.Prometheus.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrPrometheus).Value + d.global.Prometheus.Port = d.mapper.Get(ingtypes.GlobalPrometheusPort).Int() + // stats d.global.Stats.AcceptProxy = d.mapper.Get(ingtypes.GlobalStatsProxyProtocol).Bool() d.global.Stats.Auth = d.mapper.Get(ingtypes.GlobalStatsAuth).Value d.global.Stats.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrStats).Value @@ -186,11 +193,6 @@ func (c *updater) buildGlobalSSL(d *globalData) { ssl.HeadersPrefix = d.mapper.Get(ingtypes.GlobalSSLHeadersPrefix).Value } -func (c *updater) buildGlobalHealthz(d *globalData) { - d.global.Healthz.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrHealthz).Value - d.global.Healthz.Port = d.mapper.Get(ingtypes.GlobalHealthzPort).Int() -} - func (c *updater) buildGlobalHTTPStoHTTP(d *globalData) { bind := d.mapper.Get(ingtypes.GlobalBindFrontingProxy).Value if bind == "" { diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index e13229a1e..94b76c08d 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -120,7 +120,6 @@ func (c *updater) UpdateGlobalConfig(haproxyConfig haproxy.Config, mapper *Mappe c.buildGlobalCustomConfig(d) c.buildGlobalDNS(d) c.buildGlobalForwardFor(d) - c.buildGlobalHealthz(d) c.buildGlobalHTTPStoHTTP(d) c.buildGlobalModSecurity(d) c.buildGlobalProc(d) diff --git a/pkg/converters/ingress/defaults.go b/pkg/converters/ingress/defaults.go index 926e49335..f1d16db7f 100644 --- a/pkg/converters/ingress/defaults.go +++ b/pkg/converters/ingress/defaults.go @@ -62,10 +62,6 @@ func createDefaults() map[string]string { types.BackWAFMode: "deny", // types.GlobalAcmeExpiring: "30", - types.GlobalBindIPAddrHealthz: "*", - types.GlobalBindIPAddrHTTP: "*", - types.GlobalBindIPAddrStats: "*", - types.GlobalBindIPAddrTCP: "*", types.GlobalCookieKey: "Ingress", types.GlobalDNSAcceptedPayloadSize: "8192", types.GlobalDNSClusterDomain: "cluster.local", @@ -84,6 +80,7 @@ func createDefaults() map[string]string { types.GlobalNbprocBalance: "1", types.GlobalNbthread: "2", types.GlobalNoTLSRedirectLocations: "/.well-known/acme-challenge", + types.GlobalPrometheusPort: "9100", types.GlobalSSLCiphers: defaultSSLCiphers, types.GlobalSSLCipherSuites: defaultSSLCipherSuites, types.GlobalSSLDHDefaultMaxSize: "2048", diff --git a/pkg/converters/ingress/types/global.go b/pkg/converters/ingress/types/global.go index b5c65aa64..e0113f9c9 100644 --- a/pkg/converters/ingress/types/global.go +++ b/pkg/converters/ingress/types/global.go @@ -28,6 +28,7 @@ const ( GlobalBindHTTPS = "bind-https" GlobalBindIPAddrHealthz = "bind-ip-addr-healthz" GlobalBindIPAddrHTTP = "bind-ip-addr-http" + GlobalBindIPAddrPrometheus = "bind-ip-addr-prometheus" GlobalBindIPAddrStats = "bind-ip-addr-stats" GlobalBindIPAddrTCP = "bind-ip-addr-tcp" GlobalConfigDefaults = "config-defaults" @@ -60,6 +61,7 @@ const ( GlobalNbprocSSL = "nbproc-ssl" GlobalNbthread = "nbthread" GlobalNoTLSRedirectLocations = "no-tls-redirect-locations" + GlobalPrometheusPort = "prometheus-port" GlobalSSLCiphers = "ssl-ciphers" GlobalSSLCipherSuites = "ssl-cipher-suites" GlobalSSLDHDefaultMaxSize = "ssl-dh-default-max-size" diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index 4ea815acf..bada7ca22 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -2368,11 +2368,13 @@ backend _acme_challenge } } -func TestStatsHealthz(t *testing.T) { +func TestStats(t *testing.T) { testCases := []struct { stats hatypes.StatsConfig + prom hatypes.PromConfig healtz hatypes.HealthzConfig expectedStats string + expectedProm string expectedHealtz string }{ // 0 @@ -2415,10 +2417,25 @@ func TestStatsHealthz(t *testing.T) { }, expectedHealtz: "127.0.0.1:10253", }, + // 5 + { + prom: hatypes.PromConfig{ + Port: 9100, + }, + expectedProm: ` +frontend prometheus + mode http + bind :9100 + http-request use-service prometheus-exporter if { path /metrics } + http-request use-service lua.send-prometheus-root if { path / } + http-request use-service lua.send-404 + no log`, + }, } for _, test := range testCases { c := setup(t) c.config.Global().Stats = test.stats + c.config.Global().Prometheus = test.prom c.config.Global().Healthz = test.healtz if test.expectedStats == "" { test.expectedStats = "\n bind :0" @@ -2436,11 +2453,12 @@ listen stats stats uri / no log option httpclose - stats show-legends + stats show-legends` + test.expectedProm + ` frontend healthz mode http bind ` + test.expectedHealtz + ` monitor-uri /healthz + http-request use-service lua.send-404 no log `) c.logger.CompareLogging(defaultLogging) @@ -3029,6 +3047,7 @@ frontend healthz mode http bind :10253 monitor-uri /healthz + http-request use-service lua.send-404 no log`, } for { diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 9b8e17473..13adacc67 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -53,6 +53,7 @@ type Global struct { LoadServerState bool AdminSocket string Healthz HealthzConfig + Prometheus PromConfig Stats StatsConfig StrictHost bool UseChroot bool @@ -168,6 +169,12 @@ type HealthzConfig struct { Port int } +// PromConfig ... +type PromConfig struct { + BindIP string + Port int +} + // StatsConfig ... type StatsConfig struct { AcceptProxy bool diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index 0c756a34e..6021897c5 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -1039,6 +1039,21 @@ listen stats option httpclose stats show-legends +{{- if $global.Prometheus.Port }} + + # # # # # # # # # # # # # # # # # # # +# # +# Prometheus metrics +# +frontend prometheus + mode http + bind {{ $global.Prometheus.BindIP }}:{{ $global.Prometheus.Port }} + http-request use-service prometheus-exporter if { path /metrics } + http-request use-service lua.send-prometheus-root if { path / } + http-request use-service lua.send-404 + no log +{{- end }} + # # # # # # # # # # # # # # # # # # # # # # Monitor URI @@ -1047,6 +1062,7 @@ frontend healthz mode http bind {{ $global.Healthz.BindIP }}:{{ $global.Healthz.Port }} monitor-uri /healthz + http-request use-service lua.send-404 no log {{- if $global.ModSecurity.Endpoints }} diff --git a/rootfs/usr/local/etc/haproxy/lua/services.lua b/rootfs/usr/local/etc/haproxy/lua/services.lua index 1c0fca1c4..e451ed617 100644 --- a/rootfs/usr/local/etc/haproxy/lua/services.lua +++ b/rootfs/usr/local/etc/haproxy/lua/services.lua @@ -30,6 +30,16 @@ core.register_service("send-cors-preflight", "http", function(applet) applet:start_response() end) +core.register_service("send-prometheus-root", "http", function(applet) + send(applet, 200, [[ + +HAProxy Exporter +

HAProxy Exporter

+Metrics + +]]) +end) + core.register_service("send-404", "http", function(applet) send(applet, 404, [[

404 Not Found