Skip to content

Commit

Permalink
pkg/promtail: Initialize counters to 0 when creating client
Browse files Browse the repository at this point in the history
This ensures metrics are not missing before they're incremented the first
time, as recommended by the upstream Prometheus instrumentation guidelines:
https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics.

Currently, in the case of dropped_entries_total and dropped_bytes_total,
the metric is simply missing until the first occurrence of an entry
being dropped.
  • Loading branch information
chancez committed Jan 13, 2020
1 parent c1fe0fd commit cdf7083
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 0 deletions.
10 changes: 10 additions & 0 deletions pkg/promtail/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ var (
Name: "request_duration_seconds",
Help: "Duration of send requests.",
}, []string{"status_code", "host"})

countersWithHost = []*prometheus.CounterVec{
encodedBytes, sentBytes, droppedBytes, sentEntries, droppedEntries,
}
)

func init() {
Expand Down Expand Up @@ -129,6 +133,12 @@ func New(cfg Config, logger log.Logger) (Client, error) {

c.client.Timeout = cfg.Timeout

// Initialize counters to 0 so the metrics are exported before the first
// occurrence of incrementing to avoid missing metrics.
for _, counter := range countersWithHost {
counter.WithLabelValues(c.cfg.URL.Host).Add(0)
}

c.wg.Add(1)
go c.run()
return c, nil
Expand Down
18 changes: 18 additions & 0 deletions pkg/promtail/client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ func TestClient_Handle(t *testing.T) {
# HELP promtail_sent_entries_total Number of log entries sent to the ingester.
# TYPE promtail_sent_entries_total counter
promtail_sent_entries_total{host="__HOST__"} 3.0
# HELP promtail_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
# TYPE promtail_dropped_entries_total counter
promtail_dropped_entries_total{host="__HOST__"} 0
`,
},
"batch log entries together until the batch wait time is reached": {
Expand All @@ -93,6 +96,9 @@ func TestClient_Handle(t *testing.T) {
# HELP promtail_sent_entries_total Number of log entries sent to the ingester.
# TYPE promtail_sent_entries_total counter
promtail_sent_entries_total{host="__HOST__"} 2.0
# HELP promtail_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
# TYPE promtail_dropped_entries_total counter
promtail_dropped_entries_total{host="__HOST__"} 0
`,
},
"retry send a batch up to backoff's max retries in case the server responds with a 5xx": {
Expand All @@ -119,6 +125,9 @@ func TestClient_Handle(t *testing.T) {
# HELP promtail_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
# TYPE promtail_dropped_entries_total counter
promtail_dropped_entries_total{host="__HOST__"} 1.0
# HELP promtail_sent_entries_total Number of log entries sent to the ingester.
# TYPE promtail_sent_entries_total counter
promtail_sent_entries_total{host="__HOST__"} 0
`,
},
"do not retry send a batch in case the server responds with a 4xx": {
Expand All @@ -137,6 +146,9 @@ func TestClient_Handle(t *testing.T) {
# HELP promtail_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
# TYPE promtail_dropped_entries_total counter
promtail_dropped_entries_total{host="__HOST__"} 1.0
# HELP promtail_sent_entries_total Number of log entries sent to the ingester.
# TYPE promtail_sent_entries_total counter
promtail_sent_entries_total{host="__HOST__"} 0
`,
},
"batch log entries together honoring the client tenant ID": {
Expand All @@ -156,6 +168,9 @@ func TestClient_Handle(t *testing.T) {
# HELP promtail_sent_entries_total Number of log entries sent to the ingester.
# TYPE promtail_sent_entries_total counter
promtail_sent_entries_total{host="__HOST__"} 2.0
# HELP promtail_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
# TYPE promtail_dropped_entries_total counter
promtail_dropped_entries_total{host="__HOST__"} 0
`,
},
"batch log entries together honoring the tenant ID overridden while processing the pipeline stages": {
Expand Down Expand Up @@ -183,6 +198,9 @@ func TestClient_Handle(t *testing.T) {
# HELP promtail_sent_entries_total Number of log entries sent to the ingester.
# TYPE promtail_sent_entries_total counter
promtail_sent_entries_total{host="__HOST__"} 4.0
# HELP promtail_dropped_entries_total Number of log entries dropped because failed to be sent to the ingester after all retries.
# TYPE promtail_dropped_entries_total counter
promtail_dropped_entries_total{host="__HOST__"} 0
`,
},
}
Expand Down

0 comments on commit cdf7083

Please sign in to comment.