Merge pull request #486 from jcmoraisjr/jm-prom

Add frontend to the internal prometheus exporter
jcmoraisjr · Dec 23, 2019 · 136cf9f · 136cf9f
2 parents 4f37e9c + d50ed18
commit 136cf9f
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 24 deletions.
diff --git a/docs/content/en/docs/configuration/keys.md b/docs/content/en/docs/configuration/keys.md
@@ -112,10 +112,11 @@ The table below describes all supported configuration keys.
 | [`bind-fronting-proxy`](#bind)                       | ip + port                               | Global  |                    |
 | [`bind-http`](#bind)                                 | ip + port                               | Global  |                    |
 | [`bind-https`](#bind)                                | ip + port                               | Global  |                    |
-| [`bind-ip-addr-healthz`](#bind-ip-addr)              | IP address                              | Global  | `*`                |
-| [`bind-ip-addr-http`](#bind-ip-addr)                 | IP address                              | Global  | `*`                |
-| [`bind-ip-addr-stats`](#bind-ip-addr)                | IP address                              | Global  | `*`                |
-| [`bind-ip-addr-tcp`](#bind-ip-addr)                  | IP address                              | Global  | `*`                |
+| [`bind-ip-addr-healthz`](#bind-ip-addr)              | IP address                              | Global  |                    |
+| [`bind-ip-addr-http`](#bind-ip-addr)                 | IP address                              | Global  |                    |
+| [`bind-ip-addr-prometheus`](#bind-ip-addr)           | IP address                              | Global  |                    |
+| [`bind-ip-addr-stats`](#bind-ip-addr)                | IP address                              | Global  |                    |
+| [`bind-ip-addr-tcp`](#bind-ip-addr)                  | IP address                              | Global  |                    |
 | [`blue-green-balance`](#blue-green)                  | label=value=weight,...                  | Backend |                    |
 | [`blue-green-cookie`](#blue-green)                   | `CookieName:LabelName` pair             | Backend |                    |
 | [`blue-green-deploy`](#blue-green)                   | label=value=weight,...                  | Backend |                    |
@@ -179,6 +180,7 @@ The table below describes all supported configuration keys.
 | [`oauth`](#oauth)                                    | "oauth2_proxy"                          | Backend |                    |
 | [`oauth-headers`](#oauth)                            | `<header>:<var>,...`                    | Backend |                    |
 | [`oauth-uri-prefix`](#oauth)                         | URI prefix                              | Backend |                    |
+| [`prometheus-port`](#bind-port)                      | port number                             | Global  | `9100`             |
 | [`proxy-body-size`](#proxy-body-size)                | size (bytes)                            | Backend | unlimited          |
 | [`proxy-protocol`](#proxy-protocol)                  | [v1\|v2\|v2-ssl\|v2-ssl-cn]             | Backend |                    |
 | [`rewrite-target`](#rewrite-target)                  | path string                             | Backend |                    |
@@ -524,24 +526,29 @@ See also:
 
 ## Bind IP addr
 
-| Configuration key      | Scope    | Default | Since |
-|------------------------|----------|---------|-------|
-| `bind-ip-addr-healthz` | `Global` | `*`     |       |
-| `bind-ip-addr-http`    | `Global` | `*`     |       |
-| `bind-ip-addr-stats`   | `Global` | `*`     |       |
-| `bind-ip-addr-tcp`     | `Global` | `*`     |       |
+| Configuration key         | Scope    | Default | Since |
+|---------------------------|----------|---------|-------|
+| `bind-ip-addr-healthz`    | `Global` |         |       |
+| `bind-ip-addr-http`       | `Global` |         |       |
+| `bind-ip-addr-prometheus` | `Global` |         | v0.10 |
+| `bind-ip-addr-stats`      | `Global` |         |       |
+| `bind-ip-addr-tcp`        | `Global` |         |       |
 
-Define listening IPv4/IPv6 address on public HAProxy frontends.
+Define listening IPv4/IPv6 address on public HAProxy frontends. Since v0.10 the default
+value changed from `*` to an empty string, which haproxy interprets in the same way and
+binds on all IPv4 address.
 
 * `bind-ip-addr-tcp`: IP address of all TCP services declared on [`tcp-services`](#tcp-services-configmap) command-line option.
-* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared.
 * `bind-ip-addr-healthz`: IP address of the health check URL.
+* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared.
+* `bind-ip-addr-prometheus`: IP address of the haproxy's internal Prometheus exporter.
 * `bind-ip-addr-stats`: IP address of the statistics page. See also [`stats-port`](#stats).
 
 See also:
 
 * https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#4-bind
 * [Bind](#bind)
+* [Bind port](#bind-port)
 
 ---
 
@@ -552,10 +559,21 @@ See also:
 | `healthz-port`    | `Global` | `10253` |       |
 | `http-port`       | `Global` | `80`    |       |
 | `https-port`      | `Global` | `443`   |       |
+| `prometheus-port` | `Global` | `9100`  | v0.10 |
 
 * `healthz-port`: Define the port number HAProxy should listen to in order to answer for health checking requests. Use `/healthz` as the request path.
 * `http-port`: Define the port number of unencripted HTTP connections.
 * `https-port`: Define the port number of encripted HTTPS connections.
+* `prometheus-port`: Define the port number of the haproxy's internal Prometheus exporter. This port can be changed to zero `0` to remove the listener. A listener without being scraped does not use system resources, except for the listening port.
+
+{{% alert title="Note" %}}
+The internal Prometheus exporter runs concurrently with request processing, and it is
+about 5x slower and 20x more verbose than the CSV exporter. See the haproxy's exporter
+[doc](https://github.com/haproxy/haproxy/blob/v2.0.0/contrib/prometheus-exporter/README#L44).
+Consider use Prometheus' [haproxy_exporter](https://github.com/prometheus/haproxy_exporter)
+on very large clusters - Prometheus' implementation reads the CSV from the stats page and
+converts to the Prometheus syntax outside the haproxy process.
+{{% /alert %}}
 
 See also:
 

diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go
@@ -128,6 +128,13 @@ func (c *updater) buildGlobalProc(d *globalData) {
 }
 
 func (c *updater) buildGlobalStats(d *globalData) {
+	// healthz
+	d.global.Healthz.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrHealthz).Value
+	d.global.Healthz.Port = d.mapper.Get(ingtypes.GlobalHealthzPort).Int()
+	// prometheus
+	d.global.Prometheus.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrPrometheus).Value
+	d.global.Prometheus.Port = d.mapper.Get(ingtypes.GlobalPrometheusPort).Int()
+	// stats
 	d.global.Stats.AcceptProxy = d.mapper.Get(ingtypes.GlobalStatsProxyProtocol).Bool()
 	d.global.Stats.Auth = d.mapper.Get(ingtypes.GlobalStatsAuth).Value
 	d.global.Stats.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrStats).Value
@@ -186,11 +193,6 @@ func (c *updater) buildGlobalSSL(d *globalData) {
 	ssl.HeadersPrefix = d.mapper.Get(ingtypes.GlobalSSLHeadersPrefix).Value
 }
 
-func (c *updater) buildGlobalHealthz(d *globalData) {
-	d.global.Healthz.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrHealthz).Value
-	d.global.Healthz.Port = d.mapper.Get(ingtypes.GlobalHealthzPort).Int()
-}
-
 func (c *updater) buildGlobalHTTPStoHTTP(d *globalData) {
 	bind := d.mapper.Get(ingtypes.GlobalBindFrontingProxy).Value
 	if bind == "" {

diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go
@@ -120,7 +120,6 @@ func (c *updater) UpdateGlobalConfig(haproxyConfig haproxy.Config, mapper *Mappe
 	c.buildGlobalCustomConfig(d)
 	c.buildGlobalDNS(d)
 	c.buildGlobalForwardFor(d)
-	c.buildGlobalHealthz(d)
 	c.buildGlobalHTTPStoHTTP(d)
 	c.buildGlobalModSecurity(d)
 	c.buildGlobalProc(d)

diff --git a/pkg/converters/ingress/defaults.go b/pkg/converters/ingress/defaults.go
@@ -62,10 +62,6 @@ func createDefaults() map[string]string {
 		types.BackWAFMode:                "deny",
 		//
 		types.GlobalAcmeExpiring:                 "30",
-		types.GlobalBindIPAddrHealthz:            "*",
-		types.GlobalBindIPAddrHTTP:               "*",
-		types.GlobalBindIPAddrStats:              "*",
-		types.GlobalBindIPAddrTCP:                "*",
 		types.GlobalCookieKey:                    "Ingress",
 		types.GlobalDNSAcceptedPayloadSize:       "8192",
 		types.GlobalDNSClusterDomain:             "cluster.local",
@@ -84,6 +80,7 @@ func createDefaults() map[string]string {
 		types.GlobalNbprocBalance:                "1",
 		types.GlobalNbthread:                     "2",
 		types.GlobalNoTLSRedirectLocations:       "/.well-known/acme-challenge",
+		types.GlobalPrometheusPort:               "9100",
 		types.GlobalSSLCiphers:                   defaultSSLCiphers,
 		types.GlobalSSLCipherSuites:              defaultSSLCipherSuites,
 		types.GlobalSSLDHDefaultMaxSize:          "2048",

diff --git a/pkg/converters/ingress/types/global.go b/pkg/converters/ingress/types/global.go
@@ -28,6 +28,7 @@ const (
 	GlobalBindHTTPS                    = "bind-https"
 	GlobalBindIPAddrHealthz            = "bind-ip-addr-healthz"
 	GlobalBindIPAddrHTTP               = "bind-ip-addr-http"
+	GlobalBindIPAddrPrometheus         = "bind-ip-addr-prometheus"
 	GlobalBindIPAddrStats              = "bind-ip-addr-stats"
 	GlobalBindIPAddrTCP                = "bind-ip-addr-tcp"
 	GlobalConfigDefaults               = "config-defaults"
@@ -60,6 +61,7 @@ const (
 	GlobalNbprocSSL                    = "nbproc-ssl"
 	GlobalNbthread                     = "nbthread"
 	GlobalNoTLSRedirectLocations       = "no-tls-redirect-locations"
+	GlobalPrometheusPort               = "prometheus-port"
 	GlobalSSLCiphers                   = "ssl-ciphers"
 	GlobalSSLCipherSuites              = "ssl-cipher-suites"
 	GlobalSSLDHDefaultMaxSize          = "ssl-dh-default-max-size"

diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go
@@ -2368,11 +2368,13 @@ backend _acme_challenge
 	}
 }
 
-func TestStatsHealthz(t *testing.T) {
+func TestStats(t *testing.T) {
 	testCases := []struct {
 		stats          hatypes.StatsConfig
+		prom           hatypes.PromConfig
 		healtz         hatypes.HealthzConfig
 		expectedStats  string
+		expectedProm   string
 		expectedHealtz string
 	}{
 		// 0
@@ -2415,10 +2417,25 @@ func TestStatsHealthz(t *testing.T) {
 			},
 			expectedHealtz: "127.0.0.1:10253",
 		},
+		// 5
+		{
+			prom: hatypes.PromConfig{
+				Port: 9100,
+			},
+			expectedProm: `
+frontend prometheus
+    mode http
+    bind :9100
+    http-request use-service prometheus-exporter if { path /metrics }
+    http-request use-service lua.send-prometheus-root if { path / }
+    http-request use-service lua.send-404
+    no log`,
+		},
 	}
 	for _, test := range testCases {
 		c := setup(t)
 		c.config.Global().Stats = test.stats
+		c.config.Global().Prometheus = test.prom
 		c.config.Global().Healthz = test.healtz
 		if test.expectedStats == "" {
 			test.expectedStats = "\n    bind :0"
@@ -2436,11 +2453,12 @@ listen stats
     stats uri /
     no log
     option httpclose
-    stats show-legends
+    stats show-legends` + test.expectedProm + `
 frontend healthz
     mode http
     bind ` + test.expectedHealtz + `
     monitor-uri /healthz
+    http-request use-service lua.send-404
     no log
 `)
 		c.logger.CompareLogging(defaultLogging)
@@ -3029,6 +3047,7 @@ frontend healthz
     mode http
     bind :10253
     monitor-uri /healthz
+    http-request use-service lua.send-404
     no log`,
 	}
 	for {

diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go
@@ -53,6 +53,7 @@ type Global struct {
 	LoadServerState bool
 	AdminSocket     string
 	Healthz         HealthzConfig
+	Prometheus      PromConfig
 	Stats           StatsConfig
 	StrictHost      bool
 	UseChroot       bool
@@ -168,6 +169,12 @@ type HealthzConfig struct {
 	Port   int
 }
 
+// PromConfig ...
+type PromConfig struct {
+	BindIP string
+	Port   int
+}
+
 // StatsConfig ...
 type StatsConfig struct {
 	AcceptProxy bool

diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl
@@ -1039,6 +1039,21 @@ listen stats
     option httpclose
     stats show-legends
 
+{{- if $global.Prometheus.Port }}
+
+  # # # # # # # # # # # # # # # # # # #
+# #
+#     Prometheus metrics
+#
+frontend prometheus
+    mode http
+    bind {{ $global.Prometheus.BindIP }}:{{ $global.Prometheus.Port }}
+    http-request use-service prometheus-exporter if { path /metrics }
+    http-request use-service lua.send-prometheus-root if { path / }
+    http-request use-service lua.send-404
+    no log
+{{- end }}
+
   # # # # # # # # # # # # # # # # # # #
 # #
 #     Monitor URI
@@ -1047,6 +1062,7 @@ frontend healthz
     mode http
     bind {{ $global.Healthz.BindIP }}:{{ $global.Healthz.Port }}
     monitor-uri /healthz
+    http-request use-service lua.send-404
     no log
 
 {{- if $global.ModSecurity.Endpoints }}

diff --git a/rootfs/usr/local/etc/haproxy/lua/services.lua b/rootfs/usr/local/etc/haproxy/lua/services.lua
@@ -30,6 +30,16 @@ core.register_service("send-cors-preflight", "http", function(applet)
     applet:start_response()
 end)
 
+core.register_service("send-prometheus-root", "http", function(applet)
+    send(applet, 200, [[
+<html>
+<head><title>HAProxy Exporter</title></head>
+<body><h1>HAProxy Exporter</h1>
+<a href='/metrics'>Metrics</a>
+</body></html>
+]])
+end)
+
 core.register_service("send-404", "http", function(applet)
     send(applet, 404, [[
 <html><body><h1>404 Not Found</h1>