From 26cb4bbca172b8eac2c5f60d0759e39e2ef38828 Mon Sep 17 00:00:00 2001 From: Justin Burnham Date: Fri, 13 May 2022 12:32:38 -0700 Subject: [PATCH 1/6] add the `default` sprig template function to logql label/line formatter With the `default` sprig template function, we will be able to make it easier to output a default value if the source string is otherwise empty. Useful for json fields that can be missing, like HTTP headers in a log line that aren't required as in the following: ``` {job="access_log"} | json | line_format `{{.http_request_headers_x_forwarded_for | default "-"}}` ``` Previously this could only be done (that I found) with the much more verbose ``` {{regexReplaceAllLiteral "^$" .http_request_headers_x_forwarded_for "-"}} ``` --- CHANGELOG.md | 1 + docs/sources/logql/template_functions.md | 14 ++++++++++++++ pkg/logql/log/fmt.go | 1 + pkg/logql/log/fmt_test.go | 16 ++++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1079db9cadd41..011c64dc791f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ * [4860](https://github.com/grafana/loki/pull/4860) **cyriltovena**: Add rate limiting and metrics to hedging * [4865](https://github.com/grafana/loki/pull/4865) **taisho6339**: Fix duplicate registry.MustRegister call in Promtail Kafka * [4845](https://github.com/grafana/loki/pull/4845) **chaudum** Return error responses consistently as JSON +* [](https://github.com/grafana/loki/pull/) **jburnham**: Add `default` sprig template function in logql label/line formatter ## Unreleased ### All Changes diff --git a/docs/sources/logql/template_functions.md b/docs/sources/logql/template_functions.md index aad284691f813..ac7cc3253ab43 100644 --- a/docs/sources/logql/template_functions.md +++ b/docs/sources/logql/template_functions.md @@ -666,3 +666,17 @@ Example of a query to filter Loki querier jobs which create time is 1 day before ```logql {job="loki/querier"} | label_format nowEpoch=`{{(unixEpoch now)}}`,createDateEpoch=`{{unixEpoch (toDate "2006-01-02" .createDate)}}` | label_format dateTimeDiff="{{sub .nowEpoch .createDateEpoch}}" | dateTimeDiff > 86400 ``` + +## default + +`default` checks whether the string is set, and returns default if not set. + +Signature: `default(d string,src string) string` + +Examples: + +```template +{{ .http_request_headers_x_forwarded_for | default "-" }} +{{ default "-" "" }} // output: - +{{ default "-" "foo" }} // output: foo +``` diff --git a/pkg/logql/log/fmt.go b/pkg/logql/log/fmt.go index 600d12d7ab554..1fc52f974e8ce 100644 --- a/pkg/logql/log/fmt.go +++ b/pkg/logql/log/fmt.go @@ -84,6 +84,7 @@ var ( "toDate", "now", "unixEpoch", + "default", } ) diff --git a/pkg/logql/log/fmt_test.go b/pkg/logql/log/fmt_test.go index 71d111a43d8f8..30dc02e3dbc2c 100644 --- a/pkg/logql/log/fmt_test.go +++ b/pkg/logql/log/fmt_test.go @@ -265,6 +265,14 @@ func Test_lineFormatter_Format(t *testing.T) { labels.Labels{{Name: "bar", Value: "2"}}, []byte("1"), }, + { + "default", + newMustLineFormatter(`{{.foo | default "-" }}{{.bar | default "-"}}{{.unknown | default "-"}}`), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: ""}}, + []byte("blip--"), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: ""}}, + nil, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -325,6 +333,14 @@ func Test_labelsFormatter_Format(t *testing.T) { labels.Labels{{Name: "status", Value: "200"}}, labels.Labels{{Name: "status", Value: "2"}}, }, + { + "default", + mustNewLabelsFormatter([]LabelFmt{ + NewTemplateLabelFmt("blip", `{{.foo | default "-" }} and {{.bar}}`), + }), + labels.Labels{{Name: "bar", Value: "blop"}}, + labels.Labels{{Name: "blip", Value: "- and blop"}, {Name: "bar", Value: "blop"}}, + }, } for _, tt := range tests { From 3a7990b97838b099d308eecc4eff39760027b56e Mon Sep 17 00:00:00 2001 From: Justin Burnham Date: Fri, 13 May 2022 12:49:44 -0700 Subject: [PATCH 2/6] update changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 011c64dc791f3..20e3e8a02a3c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ * [4860](https://github.com/grafana/loki/pull/4860) **cyriltovena**: Add rate limiting and metrics to hedging * [4865](https://github.com/grafana/loki/pull/4865) **taisho6339**: Fix duplicate registry.MustRegister call in Promtail Kafka * [4845](https://github.com/grafana/loki/pull/4845) **chaudum** Return error responses consistently as JSON -* [](https://github.com/grafana/loki/pull/) **jburnham**: Add `default` sprig template function in logql label/line formatter +* [](https://github.com/grafana/loki/pull/) **jburnham**: LogQL: add `default` sprig template function in logql label/line formatter ## Unreleased ### All Changes From 942db0bbbbc9d29337e8941c0af82bc9ac46c5dc Mon Sep 17 00:00:00 2001 From: Justin Burnham Date: Fri, 13 May 2022 12:55:07 -0700 Subject: [PATCH 3/6] add pr number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e3e8a02a3c7..1997ed853e2a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ * [4860](https://github.com/grafana/loki/pull/4860) **cyriltovena**: Add rate limiting and metrics to hedging * [4865](https://github.com/grafana/loki/pull/4865) **taisho6339**: Fix duplicate registry.MustRegister call in Promtail Kafka * [4845](https://github.com/grafana/loki/pull/4845) **chaudum** Return error responses consistently as JSON -* [](https://github.com/grafana/loki/pull/) **jburnham**: LogQL: add `default` sprig template function in logql label/line formatter +* [6163](https://github.com/grafana/loki/pull/6163) **jburnham**: LogQL: add `default` sprig template function in logql label/line formatter ## Unreleased ### All Changes From 7e211601119b87499100fa4af5a9e10364f8aa20 Mon Sep 17 00:00:00 2001 From: Justin Burnham Date: Fri, 13 May 2022 13:15:05 -0700 Subject: [PATCH 4/6] enhance docs --- docs/sources/logql/template_functions.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sources/logql/template_functions.md b/docs/sources/logql/template_functions.md index ac7cc3253ab43..d56844f66dd00 100644 --- a/docs/sources/logql/template_functions.md +++ b/docs/sources/logql/template_functions.md @@ -676,7 +676,11 @@ Signature: `default(d string,src string) string` Examples: ```template -{{ .http_request_headers_x_forwarded_for | default "-" }} {{ default "-" "" }} // output: - {{ default "-" "foo" }} // output: foo ``` + +Example of a query to print a `-` if the `http_request_headers_x_forwarded_for` label is empty: +```logql +{job="access_log"} | json | line_format `{{.http_request_headers_x_forwarded_for | default "-"}}` +``` From 3ab1f88e8aff0ae6718ab749ac218c508e018e2d Mon Sep 17 00:00:00 2001 From: Justin Burnham Date: Mon, 16 May 2022 03:42:11 -0700 Subject: [PATCH 5/6] Update docs/sources/logql/template_functions.md Co-authored-by: Kaviraj Kanagaraj --- docs/sources/logql/template_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/template_functions.md b/docs/sources/logql/template_functions.md index d56844f66dd00..c2ea811c1b121 100644 --- a/docs/sources/logql/template_functions.md +++ b/docs/sources/logql/template_functions.md @@ -669,7 +669,7 @@ Example of a query to filter Loki querier jobs which create time is 1 day before ## default -`default` checks whether the string is set, and returns default if not set. +`default` checks whether the string(`src`) is set, and returns default(`d`) if not set. Signature: `default(d string,src string) string` From 8610f038e303067a22ba4cf89602e60867689a88 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 16 May 2022 09:30:07 -0400 Subject: [PATCH 6/6] Update docs/sources/logql/template_functions.md Co-authored-by: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com> --- docs/sources/logql/template_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/template_functions.md b/docs/sources/logql/template_functions.md index c2ea811c1b121..9ed5784b52ff2 100644 --- a/docs/sources/logql/template_functions.md +++ b/docs/sources/logql/template_functions.md @@ -671,7 +671,7 @@ Example of a query to filter Loki querier jobs which create time is 1 day before `default` checks whether the string(`src`) is set, and returns default(`d`) if not set. -Signature: `default(d string,src string) string` +Signature: `default(d string, src string) string` Examples: