diff --git a/docs/content/en/docs/configuration/keys.md b/docs/content/en/docs/configuration/keys.md index 91e4c5e0f..5c8f52988 100644 --- a/docs/content/en/docs/configuration/keys.md +++ b/docs/content/en/docs/configuration/keys.md @@ -112,10 +112,11 @@ The table below describes all supported configuration keys. | [`bind-fronting-proxy`](#bind) | ip + port | Global | | | [`bind-http`](#bind) | ip + port | Global | | | [`bind-https`](#bind) | ip + port | Global | | -| [`bind-ip-addr-healthz`](#bind-ip-addr) | IP address | Global | `*` | -| [`bind-ip-addr-http`](#bind-ip-addr) | IP address | Global | `*` | -| [`bind-ip-addr-stats`](#bind-ip-addr) | IP address | Global | `*` | -| [`bind-ip-addr-tcp`](#bind-ip-addr) | IP address | Global | `*` | +| [`bind-ip-addr-healthz`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-http`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-prometheus`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-stats`](#bind-ip-addr) | IP address | Global | | +| [`bind-ip-addr-tcp`](#bind-ip-addr) | IP address | Global | | | [`blue-green-balance`](#blue-green) | label=value=weight,... | Backend | | | [`blue-green-cookie`](#blue-green) | `CookieName:LabelName` pair | Backend | | | [`blue-green-deploy`](#blue-green) | label=value=weight,... | Backend | | @@ -179,6 +180,7 @@ The table below describes all supported configuration keys. | [`oauth`](#oauth) | "oauth2_proxy" | Backend | | | [`oauth-headers`](#oauth) | `
:,...` | Backend | | | [`oauth-uri-prefix`](#oauth) | URI prefix | Backend | | +| [`prometheus-port`](#bind-port) | port number | Global | `9100` | | [`proxy-body-size`](#proxy-body-size) | size (bytes) | Backend | unlimited | | [`proxy-protocol`](#proxy-protocol) | [v1\|v2\|v2-ssl\|v2-ssl-cn] | Backend | | | [`rewrite-target`](#rewrite-target) | path string | Backend | | @@ -524,24 +526,29 @@ See also: ## Bind IP addr -| Configuration key | Scope | Default | Since | -|------------------------|----------|---------|-------| -| `bind-ip-addr-healthz` | `Global` | `*` | | -| `bind-ip-addr-http` | `Global` | `*` | | -| `bind-ip-addr-stats` | `Global` | `*` | | -| `bind-ip-addr-tcp` | `Global` | `*` | | +| Configuration key | Scope | Default | Since | +|---------------------------|----------|---------|-------| +| `bind-ip-addr-healthz` | `Global` | | | +| `bind-ip-addr-http` | `Global` | | | +| `bind-ip-addr-prometheus` | `Global` | | v0.10 | +| `bind-ip-addr-stats` | `Global` | | | +| `bind-ip-addr-tcp` | `Global` | | | -Define listening IPv4/IPv6 address on public HAProxy frontends. +Define listening IPv4/IPv6 address on public HAProxy frontends. Since v0.10 the default +value changed from `*` to an empty string, which haproxy interprets in the same way and +binds on all IPv4 address. * `bind-ip-addr-tcp`: IP address of all TCP services declared on [`tcp-services`](#tcp-services-configmap) command-line option. -* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared. * `bind-ip-addr-healthz`: IP address of the health check URL. +* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared. +* `bind-ip-addr-prometheus`: IP address of the haproxy's internal Prometheus exporter. * `bind-ip-addr-stats`: IP address of the statistics page. See also [`stats-port`](#stats). See also: * https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#4-bind * [Bind](#bind) +* [Bind port](#bind-port) --- @@ -552,10 +559,21 @@ See also: | `healthz-port` | `Global` | `10253` | | | `http-port` | `Global` | `80` | | | `https-port` | `Global` | `443` | | +| `prometheus-port` | `Global` | `9100` | v0.10 | * `healthz-port`: Define the port number HAProxy should listen to in order to answer for health checking requests. Use `/healthz` as the request path. * `http-port`: Define the port number of unencripted HTTP connections. * `https-port`: Define the port number of encripted HTTPS connections. +* `prometheus-port`: Define the port number of the haproxy's internal Prometheus exporter. This port can be changed to zero `0` to remove the listener. A listener without being scraped does not use system resources, except for the listening port. + +{{% alert title="Note" %}} +The internal Prometheus exporter runs concurrently with request processing, and it is +about 5x slower and 20x more verbose than the CSV exporter. See the haproxy's exporter +[doc](https://github.com/haproxy/haproxy/blob/v2.0.0/contrib/prometheus-exporter/README#L44). +Consider use Prometheus' [haproxy_exporter](https://github.com/prometheus/haproxy_exporter) +on very large clusters - Prometheus' implementation reads the CSV from the stats page and +converts to the Prometheus syntax outside the haproxy process. +{{% /alert %}} See also: diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go index 84dfcf547..05f708ee1 100644 --- a/pkg/converters/ingress/annotations/global.go +++ b/pkg/converters/ingress/annotations/global.go @@ -128,6 +128,13 @@ func (c *updater) buildGlobalProc(d *globalData) { } func (c *updater) buildGlobalStats(d *globalData) { + // healthz + d.global.Healthz.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrHealthz).Value + d.global.Healthz.Port = d.mapper.Get(ingtypes.GlobalHealthzPort).Int() + // prometheus + d.global.Prometheus.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrPrometheus).Value + d.global.Prometheus.Port = d.mapper.Get(ingtypes.GlobalPrometheusPort).Int() + // stats d.global.Stats.AcceptProxy = d.mapper.Get(ingtypes.GlobalStatsProxyProtocol).Bool() d.global.Stats.Auth = d.mapper.Get(ingtypes.GlobalStatsAuth).Value d.global.Stats.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrStats).Value @@ -186,11 +193,6 @@ func (c *updater) buildGlobalSSL(d *globalData) { ssl.HeadersPrefix = d.mapper.Get(ingtypes.GlobalSSLHeadersPrefix).Value } -func (c *updater) buildGlobalHealthz(d *globalData) { - d.global.Healthz.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrHealthz).Value - d.global.Healthz.Port = d.mapper.Get(ingtypes.GlobalHealthzPort).Int() -} - func (c *updater) buildGlobalHTTPStoHTTP(d *globalData) { bind := d.mapper.Get(ingtypes.GlobalBindFrontingProxy).Value if bind == "" { diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index e13229a1e..94b76c08d 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -120,7 +120,6 @@ func (c *updater) UpdateGlobalConfig(haproxyConfig haproxy.Config, mapper *Mappe c.buildGlobalCustomConfig(d) c.buildGlobalDNS(d) c.buildGlobalForwardFor(d) - c.buildGlobalHealthz(d) c.buildGlobalHTTPStoHTTP(d) c.buildGlobalModSecurity(d) c.buildGlobalProc(d) diff --git a/pkg/converters/ingress/defaults.go b/pkg/converters/ingress/defaults.go index 926e49335..f1d16db7f 100644 --- a/pkg/converters/ingress/defaults.go +++ b/pkg/converters/ingress/defaults.go @@ -62,10 +62,6 @@ func createDefaults() map[string]string { types.BackWAFMode: "deny", // types.GlobalAcmeExpiring: "30", - types.GlobalBindIPAddrHealthz: "*", - types.GlobalBindIPAddrHTTP: "*", - types.GlobalBindIPAddrStats: "*", - types.GlobalBindIPAddrTCP: "*", types.GlobalCookieKey: "Ingress", types.GlobalDNSAcceptedPayloadSize: "8192", types.GlobalDNSClusterDomain: "cluster.local", @@ -84,6 +80,7 @@ func createDefaults() map[string]string { types.GlobalNbprocBalance: "1", types.GlobalNbthread: "2", types.GlobalNoTLSRedirectLocations: "/.well-known/acme-challenge", + types.GlobalPrometheusPort: "9100", types.GlobalSSLCiphers: defaultSSLCiphers, types.GlobalSSLCipherSuites: defaultSSLCipherSuites, types.GlobalSSLDHDefaultMaxSize: "2048", diff --git a/pkg/converters/ingress/types/global.go b/pkg/converters/ingress/types/global.go index b5c65aa64..e0113f9c9 100644 --- a/pkg/converters/ingress/types/global.go +++ b/pkg/converters/ingress/types/global.go @@ -28,6 +28,7 @@ const ( GlobalBindHTTPS = "bind-https" GlobalBindIPAddrHealthz = "bind-ip-addr-healthz" GlobalBindIPAddrHTTP = "bind-ip-addr-http" + GlobalBindIPAddrPrometheus = "bind-ip-addr-prometheus" GlobalBindIPAddrStats = "bind-ip-addr-stats" GlobalBindIPAddrTCP = "bind-ip-addr-tcp" GlobalConfigDefaults = "config-defaults" @@ -60,6 +61,7 @@ const ( GlobalNbprocSSL = "nbproc-ssl" GlobalNbthread = "nbthread" GlobalNoTLSRedirectLocations = "no-tls-redirect-locations" + GlobalPrometheusPort = "prometheus-port" GlobalSSLCiphers = "ssl-ciphers" GlobalSSLCipherSuites = "ssl-cipher-suites" GlobalSSLDHDefaultMaxSize = "ssl-dh-default-max-size" diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index 4ea815acf..bada7ca22 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -2368,11 +2368,13 @@ backend _acme_challenge } } -func TestStatsHealthz(t *testing.T) { +func TestStats(t *testing.T) { testCases := []struct { stats hatypes.StatsConfig + prom hatypes.PromConfig healtz hatypes.HealthzConfig expectedStats string + expectedProm string expectedHealtz string }{ // 0 @@ -2415,10 +2417,25 @@ func TestStatsHealthz(t *testing.T) { }, expectedHealtz: "127.0.0.1:10253", }, + // 5 + { + prom: hatypes.PromConfig{ + Port: 9100, + }, + expectedProm: ` +frontend prometheus + mode http + bind :9100 + http-request use-service prometheus-exporter if { path /metrics } + http-request use-service lua.send-prometheus-root if { path / } + http-request use-service lua.send-404 + no log`, + }, } for _, test := range testCases { c := setup(t) c.config.Global().Stats = test.stats + c.config.Global().Prometheus = test.prom c.config.Global().Healthz = test.healtz if test.expectedStats == "" { test.expectedStats = "\n bind :0" @@ -2436,11 +2453,12 @@ listen stats stats uri / no log option httpclose - stats show-legends + stats show-legends` + test.expectedProm + ` frontend healthz mode http bind ` + test.expectedHealtz + ` monitor-uri /healthz + http-request use-service lua.send-404 no log `) c.logger.CompareLogging(defaultLogging) @@ -3029,6 +3047,7 @@ frontend healthz mode http bind :10253 monitor-uri /healthz + http-request use-service lua.send-404 no log`, } for { diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 9b8e17473..13adacc67 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -53,6 +53,7 @@ type Global struct { LoadServerState bool AdminSocket string Healthz HealthzConfig + Prometheus PromConfig Stats StatsConfig StrictHost bool UseChroot bool @@ -168,6 +169,12 @@ type HealthzConfig struct { Port int } +// PromConfig ... +type PromConfig struct { + BindIP string + Port int +} + // StatsConfig ... type StatsConfig struct { AcceptProxy bool diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index 0c756a34e..6021897c5 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -1039,6 +1039,21 @@ listen stats option httpclose stats show-legends +{{- if $global.Prometheus.Port }} + + # # # # # # # # # # # # # # # # # # # +# # +# Prometheus metrics +# +frontend prometheus + mode http + bind {{ $global.Prometheus.BindIP }}:{{ $global.Prometheus.Port }} + http-request use-service prometheus-exporter if { path /metrics } + http-request use-service lua.send-prometheus-root if { path / } + http-request use-service lua.send-404 + no log +{{- end }} + # # # # # # # # # # # # # # # # # # # # # # Monitor URI @@ -1047,6 +1062,7 @@ frontend healthz mode http bind {{ $global.Healthz.BindIP }}:{{ $global.Healthz.Port }} monitor-uri /healthz + http-request use-service lua.send-404 no log {{- if $global.ModSecurity.Endpoints }} diff --git a/rootfs/usr/local/etc/haproxy/lua/services.lua b/rootfs/usr/local/etc/haproxy/lua/services.lua index 1c0fca1c4..e451ed617 100644 --- a/rootfs/usr/local/etc/haproxy/lua/services.lua +++ b/rootfs/usr/local/etc/haproxy/lua/services.lua @@ -30,6 +30,16 @@ core.register_service("send-cors-preflight", "http", function(applet) applet:start_response() end) +core.register_service("send-prometheus-root", "http", function(applet) + send(applet, 200, [[ + +HAProxy Exporter +

HAProxy Exporter

+Metrics + +]]) +end) + core.register_service("send-404", "http", function(applet) send(applet, 404, [[

404 Not Found