From 6715d03d9f74f2cf47e65ec8e194e942d525cfd2 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 27 Oct 2020 12:49:17 +0100 Subject: [PATCH 01/39] Add error labels on failure and new functions for templates. Signed-off-by: Cyril Tovena --- pkg/logql/log/error.go | 1 + pkg/logql/log/fmt.go | 75 ++++++++++++++++++++++++-- pkg/logql/log/fmt_test.go | 111 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 5 deletions(-) diff --git a/pkg/logql/log/error.go b/pkg/logql/log/error.go index 04dd20ecdcd15..a518b08c5e590 100644 --- a/pkg/logql/log/error.go +++ b/pkg/logql/log/error.go @@ -6,6 +6,7 @@ var ( errLogfmt = "LogfmtParserErr" errSampleExtraction = "SampleExtractionErr" errLabelFilter = "LabelFilterErr" + errTemplateFormat = "TemplateFormatErr" ErrorLabel = "__error__" ) diff --git a/pkg/logql/log/fmt.go b/pkg/logql/log/fmt.go index 231fa37719043..89d921ab443b6 100644 --- a/pkg/logql/log/fmt.go +++ b/pkg/logql/log/fmt.go @@ -14,6 +14,7 @@ var ( // Available map of functions for the text template engine. functionMap = template.FuncMap{ + // olds function deprecated. "ToLower": strings.ToLower, "ToUpper": strings.ToUpper, "Replace": strings.Replace, @@ -23,6 +24,26 @@ var ( "TrimPrefix": strings.TrimPrefix, "TrimSuffix": strings.TrimSuffix, "TrimSpace": strings.TrimSpace, + + // new function ported from https://github.com/Masterminds/sprig/ + "lower": strings.ToLower, + "upper": strings.ToUpper, + "title": strings.Title, + "trunc": trunc, + "substr": substring, + "contains": func(substr string, str string) bool { return strings.Contains(str, substr) }, + "hasPrefix": func(substr string, str string) bool { return strings.HasPrefix(str, substr) }, + "hasSuffix": func(substr string, str string) bool { return strings.HasSuffix(str, substr) }, + "indent": indent, + "nindent": nindent, + "replace": replace, + "repeat": func(count int, str string) string { return strings.Repeat(str, count) }, + "trim": strings.TrimSpace, + // Switch order so that "$foo" | trimall "$" + "trimAll": func(a, b string) string { return strings.Trim(b, a) }, + "trimSuffix": func(a, b string) string { return strings.TrimSuffix(b, a) }, + "trimPrefix": func(a, b string) string { return strings.TrimPrefix(b, a) }, + "regexReplaceAll": func(regex string, s string, repl string) string { r := regexp.MustCompile(regex) return r.ReplaceAllString(s, repl) @@ -51,10 +72,12 @@ func NewFormatter(tmpl string) (*LineFormatter, error) { }, nil } -func (lf *LineFormatter) Process(_ []byte, lbs *LabelsBuilder) ([]byte, bool) { +func (lf *LineFormatter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { lf.buf.Reset() - // todo(cyriltovena): handle error - _ = lf.Template.Execute(lf.buf, lbs.Labels().Map()) + if err := lf.Template.Execute(lf.buf, lbs.Labels().Map()); err != nil { + lbs.SetErr(errTemplateFormat) + return line, true + } // todo(cyriltovena): we might want to reuse the input line or a bytes buffer. res := make([]byte, len(lf.buf.Bytes())) copy(res, lf.buf.Bytes()) @@ -150,12 +173,54 @@ func (lf *LabelsFormatter) Process(l []byte, lbs *LabelsBuilder) ([]byte, bool) continue } lf.buf.Reset() - //todo (cyriltovena): handle error if data == nil { data = lbs.Labels().Map() } - _ = f.tmpl.Execute(lf.buf, data) + if err := f.tmpl.Execute(lf.buf, data); err != nil { + lbs.SetErr(errTemplateFormat) + continue + } lbs.Set(f.Name, lf.buf.String()) } return l, true } + +func trunc(c int, s string) string { + if c < 0 && len(s)+c > 0 { + return s[len(s)+c:] + } + if c >= 0 && len(s) > c { + return s[:c] + } + return s +} + +// substring creates a substring of the given string. +// +// If start is < 0, this calls string[:end]. +// +// If start is >= 0 and end < 0 or end bigger than s length, this calls string[start:] +// +// Otherwise, this calls string[start, end]. +func substring(start, end int, s string) string { + if start < 0 { + return s[:end] + } + if end < 0 || end > len(s) { + return s[start:] + } + return s[start:end] +} + +func replace(old, new, src string) string { + return strings.Replace(src, old, new, -1) +} + +func indent(spaces int, v string) string { + pad := strings.Repeat(" ", spaces) + return pad + strings.Replace(v, "\n", "\n"+pad, -1) +} + +func nindent(spaces int, v string) string { + return "\n" + indent(spaces, v) +} diff --git a/pkg/logql/log/fmt_test.go b/pkg/logql/log/fmt_test.go index b86d6ad06a27e..a6e5518255fc1 100644 --- a/pkg/logql/log/fmt_test.go +++ b/pkg/logql/log/fmt_test.go @@ -24,6 +24,115 @@ func Test_lineFormatter_Format(t *testing.T) { []byte("fooblipbuzzblop"), labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, }, + { + "Replace", + newMustLineFormatter(`foo{{.foo}}buzz{{ Replace .bar "blop" "bar" -1 }}`), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + []byte("fooblipbuzzbar"), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + }, + { + "replace", + newMustLineFormatter(`foo{{.foo}}buzz{{ .bar | replace "blop" "bar" }}`), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + []byte("fooblipbuzzbar"), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + }, + { + "title", + newMustLineFormatter(`{{.foo | title }}`), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + []byte("Blip"), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + }, + { + "substr and trunc", + newMustLineFormatter( + `{{.foo | substr 1 3 }} {{ .bar | trunc 1 }} {{ .bar | trunc 3 }}`, + ), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + []byte("li b blo"), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + }, + { + "trim", + newMustLineFormatter( + `{{.foo | trim }} {{ .bar | trimAll "op" }} {{ .bar | trimPrefix "b" }} {{ .bar | trimSuffix "p" }}`, + ), + labels.Labels{{Name: "foo", Value: " blip "}, {Name: "bar", Value: "blop"}}, + []byte("blip bl lop blo"), + labels.Labels{{Name: "foo", Value: " blip "}, {Name: "bar", Value: "blop"}}, + }, + { + "lower and upper", + newMustLineFormatter(`{{.foo | lower }} {{ .bar | upper }}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("blip BLOP"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "repeat", + newMustLineFormatter(`{{ "foo" | repeat 3 }}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("foofoofoo"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "indent", + newMustLineFormatter(`{{ "foo\n bar" | indent 4 }}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte(" foo\n bar"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "nindent", + newMustLineFormatter(`{{ "foo" | nindent 2 }}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("\n foo"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "contains", + newMustLineFormatter(`{{ if .foo | contains "p"}}yes{{end}}-{{ if .foo | contains "z"}}no{{end}}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("yes-"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "hasPrefix", + newMustLineFormatter(`{{ if .foo | hasPrefix "BL" }}yes{{end}}-{{ if .foo | hasPrefix "p"}}no{{end}}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("yes-"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "hasSuffix", + newMustLineFormatter(`{{ if .foo | hasSuffix "Ip" }}yes{{end}}-{{ if .foo | hasSuffix "pw"}}no{{end}}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("yes-"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "regexReplaceAll", + newMustLineFormatter(`{{ regexReplaceAll "(p)" .foo "t" }}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("BLIt"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "regexReplaceAllLiteral", + newMustLineFormatter(`{{ regexReplaceAllLiteral "(p)" .foo "${1}" }}`), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + []byte("BLI${1}"), + labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}}, + }, + { + "err", + newMustLineFormatter(`{{.foo Replace "foo"}}`), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + nil, + labels.Labels{{Name: ErrorLabel, Value: errTemplateFormat}, {Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + }, { "missing", newMustLineFormatter("foo {{.foo}}buzz{{ .bar }}"), @@ -41,6 +150,8 @@ func Test_lineFormatter_Format(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + sort.Sort(tt.lbs) + sort.Sort(tt.wantLbs) b := NewLabelsBuilder() b.Reset(tt.lbs) outLine, _ := tt.fmter.Process(nil, b) From 94adcd9142c44953c832a14ccc4647be8267d7df Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 27 Oct 2020 14:54:22 +0100 Subject: [PATCH 02/39] Updates docs. Signed-off-by: Cyril Tovena --- docs/sources/logql/_index.md | 79 +------ docs/sources/logql/functions.md | 370 ++++++++++++++++++++++++++++++++ 2 files changed, 372 insertions(+), 77 deletions(-) create mode 100644 docs/sources/logql/functions.md diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index 3c2a83ff1a740..4a19133c726d8 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -295,7 +295,7 @@ Will extract and rewrite the log line to only contains the query and the duratio You can use double quoted string for the template or single backtick \``\{{.label_name}}`\` to avoid the need to escape special characters. -See [functions](#Template-functions) to learn about available functions in the template format. +See [functions](functions/) to learn about available functions in the template format. #### Labels Format Expression @@ -303,7 +303,7 @@ The `| label_format` expression can renamed, modify or add labels. It takes as p When both side are label identifiers, for example `dst=src`, the operation will rename the `src` label into `dst`. -The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value will be replace by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, this mean labels are available as variables and you can use the same list of [functions](#Template-functions). +The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value will be replace by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, this mean labels are available as variables and you can use the same list of [functions](functions/). In both case if the destination label doesn't exist a new one will be created. @@ -311,81 +311,6 @@ The renaming form `dst=src` will _drop_ the `src` label after remapping it to th > A single label name can only appear once per expression. This means `| label_format foo=bar,foo="new"` is not allowed but you can use two expressions for the desired effect: `| label_format foo=bar | label_format foo="new"` -#### Template functions - -The text template format used in `| line_format` and `| label_format` support functions the following list of functions. - -##### ToLower & ToUpper - -Convert the entire string to lowercase or uppercase: - -Examples: - -```template -"{{.request_method | ToLower}}" -"{{.request_method | ToUpper}}" -`{{ToUpper "This is a string" | ToLower}}` -``` - -##### Replace - -Perform simple string replacement. - -It takes three arguments: - -- string to replace -- string to replace with -- source string - -Example: - -```template -`"This is a string" | Replace " " "-"` -``` - -The above will produce `This-is-a-string` - -##### Trim - -`Trim` returns a slice of the string s with all leading and -trailing Unicode code points contained in cutset removed. - -`TrimLeft` and `TrimRight` are the same as `Trim` except that it respectively trim only leading and trailing characters. - -```template -`{{ Trim .query ",. " }}` -`{{ TrimLeft .uri ":" }}` -`{{ TrimRight .path "/" }}` -``` - -`TrimSpace` TrimSpace returns string s with all leading -and trailing white space removed, as defined by Unicode. - -```template -{{ TrimSpace .latency }} -``` - -`TrimPrefix` and `TrimSuffix` will trim respectively the prefix or suffix supplied. - -```template -{{ TrimPrefix .path "/" }} -``` - -##### Regex - -`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [docs](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for detailed examples. - -```template -`{{ regexReplaceAllLiteral "(a*)bc" .some_label "${1}a" }}` -``` - -`regexReplaceAllLiteral` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement The replacement string is substituted directly, without using Expand. - -```template -`{{ regexReplaceAllLiteral "(ts=)" .timestamp "timestamp=" }}` -``` - -You can combine multiple function using pipe, for example if you want to strip out spaces and make the request method in capital you would write the following template `{{ .request_method | TrimSpace | ToUpper }}`. ### Log Queries Examples diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md new file mode 100644 index 0000000000000..3147a18438f7a --- /dev/null +++ b/docs/sources/logql/functions.md @@ -0,0 +1,370 @@ +--- +title: Template functions +--- + +The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support functions the usage of functions. + +All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`) for example the following template will output the value of the path label: + +```template +{{ .path }} +``` + +You can take advantage of [pipeline](https://golang.org/pkg/text/template/#hdr-Pipelines) to chain multiple functions. +In a chained pipeline, the result of each command is passed as the last argument of the following command. + +Example: + +```template +{{ .path | replace " " "_" | trunc 5 | upper }} +``` + +## ToLower & ToUpper + +Convert the entire string to lowercase or uppercase. + +Signatures: + +- `ToLower(string) string` +- `ToUpper(string) string` + +Examples: + +```template +"{{.request_method | ToLower}}" +"{{.request_method | ToUpper}}" +`{{ToUpper "This is a string" | ToLower}}` +``` + +> In Loki 2.1 you can also use respectively [`lower`](#lower) and [`upper`](#upper) shortcut, e.g `{{.request_method | lower }}` + +## Replace string + +> in Loki 2.1 [`replace`](#replace) (as opposed to `Replace`) is available with a different signature but easier to chain within pipeline. + +Perform simple string replacement. + +Signature: + +`Replace(s, old, new string, n int) string` + +It takes four arguments: + +- `s` source string +- `old` string to replace +- `new` string to replace with +- `n` the maximun amount of replacement (-1 for all) + +Example: + +```template +`{{ Replace "This is a string" " " "-" -1 }}` +``` + +The above will produce `This-is-a-string` + +## Trim, TrimLeft, TrimRight and TrimSpace + +> In Loki 2.1 [trim](#trim), [trimAll](#trimAll), [trimSuffix](#trimSuffix) and [trimPrefix](trimPrefix) have been added with a different signature for better pipeline chaining. + +`Trim` returns a slice of the string s with all leading and +trailing Unicode code points contained in cutset removed. + +Signature: `Trim(value, cutset string) string` + +`TrimLeft` and `TrimRight` are the same as `Trim` except that it respectively trim only leading and trailing characters. + +```template +`{{ Trim .query ",. " }}` +`{{ TrimLeft .uri ":" }}` +`{{ TrimRight .path "/" }}` +``` + +`TrimSpace` TrimSpace returns string s with all leading +and trailing white space removed, as defined by Unicode. + +Signature: `TrimSpace(value string) string` + +```template +{{ TrimSpace .latency }} +``` + +`TrimPrefix` and `TrimSuffix` will trim respectively the prefix or suffix supplied. + +Signature: + +- `TrimPrefix(value string, prefix string) string` +- `TrimSuffix(value string, suffix string) string` + +```template +{{ TrimPrefix .path "/" }} +``` + +## regexReplaceAll & regexReplaceAllLiteral + +`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [docs](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for detailed examples. + +```template +`{{ regexReplaceAllLiteral "(a*)bc" .some_label "${1}a" }}` +``` + +`regexReplaceAllLiteral` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement The replacement string is substituted directly, without using Expand. + +```template +`{{ regexReplaceAllLiteral "(ts=)" .timestamp "timestamp=" }}` +``` + +You can combine multiple function using pipe, for example if you want to strip out spaces and make the request method in capital you would write the following template `{{ .request_method | TrimSpace | ToUpper }}`. + +## lower + +> Added in Loki 2.1 + +Convert to lower case. + +Signature: + +`lower(string) string` + +Examples: + +```template +"{{ .request_method | lower }}" +`{{ lower "HELLO"}}` +``` + +The last example will return `hello`. + +## upper + +> Added in Loki 2.1 + +Convert to upper case. + +Signature: + +`upper(string) string` + +Examples: + +```template +"{{ .request_method | upper }}" +`{{ upper "hello"}}` +``` + +The last example will return `HELLO`. + +## title + +> Added in Loki 2.1 + +Convert to title case. + +Signature: + +`title(string) string` + +Examples: + +```template +"{{.request_method | title}}" +`{{ title "hello world"}}` +``` + +The last example will return `Hello World`. + +## trunc + +> Added in Loki 2.1 + +Truncate a string (and add no suffix) + +Signature: + +`trunc(count int,value string) string` + +Examples: + +```template +"{{ .path | trunc 2 }}" +`{{ trunc 5 "hello world"}}` // output: hello +`{{ trunc -5 "hello world"}}` // output: world +``` + +## substr + +> Added in Loki 2.1 + +Get a substring from a string. + +Signature: + +`trunc(start int,end int,value string) string` + +If start is < 0, this calls value[:end]. +If start is >= 0 and end < 0 or end bigger than s length, this calls value[start:] +Otherwise, this calls value[start, end]. + +Examples: + +```template +"{{ .path | substr 2 5 }}" +`{{ substr 0 5 "hello world"}}` // output: hello +`{{ substr 6 11 "hello world"}}` // output: world +``` + +## replace + +> Added in Loki 2.1 + +Perform simple string replacement. + +Signature: `replace(old string, new string, src string) string` + +It takes three arguments: + +- `old` string to replace +- `new` string to replace with +- `src` source string + +Examples: + +```template +{{ .cluster | replace "-cluster" "" }} +{{ replace "hello" "world" "hello world" }} +``` + +The last example will return `world world`. + +## trim + +> Added in Loki 2.1 + +The trim function removes space from either side of a string. + +Signature: `trim(string) string` + +Examples: + +```template +{{ .ip | trim }} +{{ trim " hello " }} // output: hello +``` + +## trimAll + +> Added in Loki 2.1 + +Remove given characters from the front or back of a string. + +Signature: `trimAll(chars string,src string) string` + +Examples: + +```template +{{ .path | trimAll "/" }} +{{ trimAll "$" "$5.00" }} // output: 5.00 +``` + +## trimSuffix + +> Added in Loki 2.1 + +Trim just the suffix from a string. + +Signature: `trimSuffix(suffix string, src string) string` + +Examples: + +```template +{{ .path | trimSuffix "/" }} +{{ trimSuffix "-" "hello-" }} // output: hello +``` + +## trimPrefix + +> Added in Loki 2.1 + +Trim just the prefix from a string. + +Signature: `trimPrefix(suffix string, src string) string` + +Examples: + +```template +{{ .path | trimPrefix "/" }} +{{ trimPrefix "-" "-hello" }} // output: hello +``` + +## indent + +> Added in Loki 2.1 + +The indent function indents every line in a given string to the specified indent width. This is useful when aligning multi-line strings. + +Signature: `indent(spaces int,src string) string` + +```template +{{ indent 4 .query }} +``` + +The following will indent by 4 spaces each lines contained in `.query`. + +## nindent + +> Added in Loki 2.1 + +The nindent function is the same as the indent function, but prepends a new line to the beginning of the string. + +Signature: `nindent(spaces int,src string) string` + +```template +{{ nindent 4 .query }} +``` + +The above will indent every line of text by 4 space characters and add a new line to the beginning. + +## repeat + +> Added in Loki 2.1 + +Repeat a string multiple times. + +Signature: `repeat(c int,value string) string` + +```template +{{ repeat 3 "hello" }} // output: hellohellohello +``` + +## contains + +> Added in Loki 2.1 + +Test to see if one string is contained inside of another. + +Signature: `contains(s string, src string) bool` + +Examples: + +```template +{{ if .err contains "ErrTimeout" }} timeout {{end}} +{{ if contains "he" "hello" }} yes {{end}} +``` + +## hasPrefix & hasSuffix + +> Added in Loki 2.1 + +The `hasPrefix` and `hasSuffix` functions test whether a string has a given prefix or suffix. + +Signatures: + +- `hasPrefix(prefix string, src string) bool` +- `hasSuffix(suffix string, src string) bool` + +Examples: + +```template +{{ if .err hasSuffix "Timeout" }} timeout {{end}} +{{ if hasPrefix "he" "hello" }} yes {{end}} +``` From f2b0e995fe1bdc95a82880a45c638dc5681d0cb6 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:11:28 -0500 Subject: [PATCH 03/39] Update docs/sources/logql/_index.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index 4a19133c726d8..f42651ac7ef85 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -303,7 +303,7 @@ The `| label_format` expression can renamed, modify or add labels. It takes as p When both side are label identifiers, for example `dst=src`, the operation will rename the `src` label into `dst`. -The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value will be replace by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, this mean labels are available as variables and you can use the same list of [functions](functions/). +The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value will be replace by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, which means labels are available as variables and you can use the same list of [functions](functions/). In both case if the destination label doesn't exist a new one will be created. From d28320b6c791cbb74421db955cca9bcf66e0f748 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:11:41 -0500 Subject: [PATCH 04/39] Update docs/sources/logql/_index.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index f42651ac7ef85..d7e9beeb0b3b5 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -305,7 +305,7 @@ When both side are label identifiers, for example `dst=src`, the operation will The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value will be replace by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, which means labels are available as variables and you can use the same list of [functions](functions/). -In both case if the destination label doesn't exist a new one will be created. +In both case if the destination label doesn't exist, then a new one is created. The renaming form `dst=src` will _drop_ the `src` label after remapping it to the `dst` label. However, the _template_ form will preserve the referenced labels, such that `dst="{{.src}}"` results in both `dst` and `src` having the same value. From b54c7a6cd387bddb8d17cb9ad1384dcf30a42676 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:11:51 -0500 Subject: [PATCH 05/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 3147a18438f7a..37de1409a166d 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -36,7 +36,7 @@ Examples: `{{ToUpper "This is a string" | ToLower}}` ``` -> In Loki 2.1 you can also use respectively [`lower`](#lower) and [`upper`](#upper) shortcut, e.g `{{.request_method | lower }}` +> **Note:** In Loki 2.1 you can also use respectively [`lower`](#lower) and [`upper`](#upper) shortcut, e.g `{{.request_method | lower }}` ## Replace string From cfec17d4e673a12908b9bd62aac2fa8c9005e27b Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:12:00 -0500 Subject: [PATCH 06/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 37de1409a166d..c14f102a0bd48 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -40,7 +40,7 @@ Examples: ## Replace string -> in Loki 2.1 [`replace`](#replace) (as opposed to `Replace`) is available with a different signature but easier to chain within pipeline. +> **Note:** In Loki 2.1 [`replace`](#replace) (as opposed to `Replace`) is available with a different signature but easier to chain within pipeline. Perform simple string replacement. From 10846379269288e1e4bedf322bc1178c61002d26 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:12:14 -0500 Subject: [PATCH 07/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index c14f102a0bd48..88a48b2e419fc 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -65,7 +65,7 @@ The above will produce `This-is-a-string` ## Trim, TrimLeft, TrimRight and TrimSpace -> In Loki 2.1 [trim](#trim), [trimAll](#trimAll), [trimSuffix](#trimSuffix) and [trimPrefix](trimPrefix) have been added with a different signature for better pipeline chaining. +> **Note:** In Loki 2.1 [trim](#trim), [trimAll](#trimAll), [trimSuffix](#trimSuffix) and [trimPrefix](trimPrefix) have been added with a different signature for better pipeline chaining. `Trim` returns a slice of the string s with all leading and trailing Unicode code points contained in cutset removed. From 2767d2436ca4793e38785fd569124790e11a7df1 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:12:27 -0500 Subject: [PATCH 08/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 88a48b2e419fc..73a46d7277608 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -61,7 +61,7 @@ Example: `{{ Replace "This is a string" " " "-" -1 }}` ``` -The above will produce `This-is-a-string` +The above will produce `This-is-a-string`. ## Trim, TrimLeft, TrimRight and TrimSpace From bf42dbc8923cbb4e93e88c4a3022a50bfa8a7842 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:12:36 -0500 Subject: [PATCH 09/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 73a46d7277608..4dbd9493e9c88 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -312,7 +312,7 @@ The following will indent by 4 spaces each lines contained in `.query`. ## nindent -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. The nindent function is the same as the indent function, but prepends a new line to the beginning of the string. From 46904dd07e731eff81dea6f9a38e583001d24500 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:12:43 -0500 Subject: [PATCH 10/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 4dbd9493e9c88..1dc2982563105 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -338,7 +338,7 @@ Signature: `repeat(c int,value string) string` ## contains -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Test to see if one string is contained inside of another. From 4a1cc243a45e530de65dcac50dd1f49dd5119dd1 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:12:51 -0500 Subject: [PATCH 11/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 1dc2982563105..28d719f0643e1 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -353,7 +353,7 @@ Examples: ## hasPrefix & hasSuffix -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. The `hasPrefix` and `hasSuffix` functions test whether a string has a given prefix or suffix. From 5ed2cff6902968540363a88e0efd9a38f5d0a078 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:13:23 -0500 Subject: [PATCH 12/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 28d719f0643e1..830cd9c9d2684 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -100,7 +100,7 @@ Signature: {{ TrimPrefix .path "/" }} ``` -## regexReplaceAll & regexReplaceAllLiteral +## regexReplaceAll and regexReplaceAllLiteral `regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [docs](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for detailed examples. From be9be60ed2685705e181247087a79e4b0e9e01f7 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:13:31 -0500 Subject: [PATCH 13/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 830cd9c9d2684..e5934fa8939f9 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -326,7 +326,7 @@ The above will indent every line of text by 4 space characters and add a new lin ## repeat -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Repeat a string multiple times. From f32becf9256957d518042b84a35f1182c6f23dba Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:13:39 -0500 Subject: [PATCH 14/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index e5934fa8939f9..011dc13d143a8 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -351,7 +351,7 @@ Examples: {{ if contains "he" "hello" }} yes {{end}} ``` -## hasPrefix & hasSuffix +## hasPrefix and hasSuffix > **Note:** Added in Loki 2.1. From 1dfda03ddc7d87e51619bf657517b04d602bda16 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:13:58 -0500 Subject: [PATCH 15/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 011dc13d143a8..142771beabeb9 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -114,7 +114,7 @@ Signature: `{{ regexReplaceAllLiteral "(ts=)" .timestamp "timestamp=" }}` ``` -You can combine multiple function using pipe, for example if you want to strip out spaces and make the request method in capital you would write the following template `{{ .request_method | TrimSpace | ToUpper }}`. +You can combine multiple function using pipe. For example, if you want to strip out spaces and make the request method in capital, then you would write the following template: `{{ .request_method | TrimSpace | ToUpper }}`. ## lower From 38ddece837c53f52196b5686eb14fc8a1caf6538 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:14:26 -0500 Subject: [PATCH 16/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 142771beabeb9..237de13d65c00 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -156,7 +156,7 @@ The last example will return `HELLO`. ## title -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Convert to title case. From 96623a7b1a7b97e3fb358f97fcdc040cf72c2129 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:14:51 -0500 Subject: [PATCH 17/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 237de13d65c00..92c850a1c94a0 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -175,7 +175,7 @@ The last example will return `Hello World`. ## trunc -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Truncate a string (and add no suffix) From a92c6615c05694622ca473def5270afcc9d676af Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:15:26 -0500 Subject: [PATCH 18/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 92c850a1c94a0..339b880f33240 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -298,7 +298,7 @@ Examples: ## indent -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. The indent function indents every line in a given string to the specified indent width. This is useful when aligning multi-line strings. From d8ad1b5d5176600214d36748859308467ca4ecf7 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:16:45 -0500 Subject: [PATCH 19/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 339b880f33240..20187d9168db9 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -177,7 +177,7 @@ The last example will return `Hello World`. > **Note:** Added in Loki 2.1. -Truncate a string (and add no suffix) +Truncate a string and add no suffix. Signature: From d24ff399bf83bae67a5bd62874454d7c012fb8ba Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:17:05 -0500 Subject: [PATCH 20/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 20187d9168db9..1a64e2113934a 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -193,7 +193,7 @@ Examples: ## substr -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Get a substring from a string. From f5d16083d5b42b0d68e280916785e63dcc309105 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:17:36 -0500 Subject: [PATCH 21/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 1a64e2113934a..9d98996fcb199 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -215,7 +215,7 @@ Examples: ## replace -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Perform simple string replacement. From dfdda5cde48d60e5b896b890bea132323404d022 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:17:48 -0500 Subject: [PATCH 22/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 9d98996fcb199..08d853c6977bd 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -238,7 +238,7 @@ The last example will return `world world`. ## trim -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. The trim function removes space from either side of a string. From 2cc17bd19956abcedee92f47235a2871e377902d Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:18:07 -0500 Subject: [PATCH 23/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 08d853c6977bd..95f29b101ef0d 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -253,7 +253,7 @@ Examples: ## trimAll -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Remove given characters from the front or back of a string. From e4f63a75c9d1825fedce8554c33300c6082fb8e3 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:18:20 -0500 Subject: [PATCH 24/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 95f29b101ef0d..e115464011560 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -268,7 +268,7 @@ Examples: ## trimSuffix -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Trim just the suffix from a string. From 565d17d3a29298d18edeae8c8493cdf9afcc8599 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 05:19:55 -0500 Subject: [PATCH 25/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index e115464011560..50b77c9be9dc4 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -283,7 +283,7 @@ Examples: ## trimPrefix -> Added in Loki 2.1 +> **Note:** Added in Loki 2.1. Trim just the prefix from a string. From 2fe24d836cc183fd05f4d7627119db86b0656cf9 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 11:21:15 +0100 Subject: [PATCH 26/39] Improve docs based off review feedback. Signed-off-by: Cyril Tovena --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 50b77c9be9dc4..fa72a0f2b6747 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -102,7 +102,7 @@ Signature: ## regexReplaceAll and regexReplaceAllLiteral -`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [docs](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for detailed examples. +`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [Regexp.replaceAll documentation](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for detailed examples. ```template `{{ regexReplaceAllLiteral "(a*)bc" .some_label "${1}a" }}` From 8c55441cdc2c424db837a2c9571ea5ada1cde905 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 12:16:54 +0100 Subject: [PATCH 27/39] Review feedback. Signed-off-by: Cyril Tovena --- pkg/logql/log/fmt.go | 88 ++++++++++++++++++++++++--------------- pkg/logql/log/fmt_test.go | 49 ++++++++++++++++++++++ 2 files changed, 103 insertions(+), 34 deletions(-) diff --git a/pkg/logql/log/fmt.go b/pkg/logql/log/fmt.go index 89d921ab443b6..b9d794bce37cc 100644 --- a/pkg/logql/log/fmt.go +++ b/pkg/logql/log/fmt.go @@ -25,25 +25,27 @@ var ( "TrimSuffix": strings.TrimSuffix, "TrimSpace": strings.TrimSpace, - // new function ported from https://github.com/Masterminds/sprig/ - "lower": strings.ToLower, - "upper": strings.ToUpper, - "title": strings.Title, - "trunc": trunc, - "substr": substring, - "contains": func(substr string, str string) bool { return strings.Contains(str, substr) }, - "hasPrefix": func(substr string, str string) bool { return strings.HasPrefix(str, substr) }, - "hasSuffix": func(substr string, str string) bool { return strings.HasSuffix(str, substr) }, - "indent": indent, - "nindent": nindent, - "replace": replace, - "repeat": func(count int, str string) string { return strings.Repeat(str, count) }, - "trim": strings.TrimSpace, - // Switch order so that "$foo" | trimall "$" - "trimAll": func(a, b string) string { return strings.Trim(b, a) }, - "trimSuffix": func(a, b string) string { return strings.TrimSuffix(b, a) }, - "trimPrefix": func(a, b string) string { return strings.TrimPrefix(b, a) }, - + // New function ported from https://github.com/Masterminds/sprig/ + // Those function takes the string as the last parameter, allowing pipe chaining. + // Example: .mylabel | lower | substring 0 5 + "lower": strings.ToLower, + "upper": strings.ToUpper, + "title": strings.Title, + "trunc": trunc, + "substr": substring, + "contains": contains, + "hasPrefix": hasPrefix, + "hasSuffix": hasSuffix, + "indent": indent, + "nindent": nindent, + "replace": replace, + "repeat": repeat, + "trim": strings.TrimSpace, + "trimAll": trimAll, + "trimSuffix": trimSuffix, + "trimPrefix": trimPrefix, + + // regex functions "regexReplaceAll": func(regex string, s string, repl string) string { r := regexp.MustCompile(regex) return r.ReplaceAllString(s, repl) @@ -186,11 +188,13 @@ func (lf *LabelsFormatter) Process(l []byte, lbs *LabelsBuilder) ([]byte, bool) } func trunc(c int, s string) string { - if c < 0 && len(s)+c > 0 { - return s[len(s)+c:] + runes := []rune(s) + l := len(s) + if c < 0 && l+c > 0 { + return string(runes[l+c:]) } - if c >= 0 && len(s) > c { - return s[:c] + if c >= 0 && l > c { + return string(runes[:c]) } return s } @@ -203,24 +207,40 @@ func trunc(c int, s string) string { // // Otherwise, this calls string[start, end]. func substring(start, end int, s string) string { + runes := []rune(s) + l := len(runes) + if end > l { + end = l + } + if start > l { + start = l + } if start < 0 { - return s[:end] + if end < 0 { + return "" + } + return string(runes[:end]) } - if end < 0 || end > len(s) { - return s[start:] + if end < 0 { + return string(runes[start:]) } - return s[start:end] -} - -func replace(old, new, src string) string { - return strings.Replace(src, old, new, -1) + if start > end { + return "" + } + return string(runes[start:end]) } +func contains(substr string, str string) bool { return strings.Contains(str, substr) } +func hasPrefix(substr string, str string) bool { return strings.HasPrefix(str, substr) } +func hasSuffix(substr string, str string) bool { return strings.HasSuffix(str, substr) } +func repeat(count int, str string) string { return strings.Repeat(str, count) } +func replace(old, new, src string) string { return strings.Replace(src, old, new, -1) } +func trimAll(a, b string) string { return strings.Trim(b, a) } +func trimSuffix(a, b string) string { return strings.TrimSuffix(b, a) } +func trimPrefix(a, b string) string { return strings.TrimPrefix(b, a) } func indent(spaces int, v string) string { pad := strings.Repeat(" ", spaces) return pad + strings.Replace(v, "\n", "\n"+pad, -1) } -func nindent(spaces int, v string) string { - return "\n" + indent(spaces, v) -} +func nindent(spaces int, v string) string { return "\n" + indent(spaces, v) } diff --git a/pkg/logql/log/fmt_test.go b/pkg/logql/log/fmt_test.go index a6e5518255fc1..0668798d3edbb 100644 --- a/pkg/logql/log/fmt_test.go +++ b/pkg/logql/log/fmt_test.go @@ -240,3 +240,52 @@ func Test_validate(t *testing.T) { }) } } + +func Test_trunc(t *testing.T) { + tests := []struct { + s string + c int + want string + }{ + {"Hello, 世界", -1, "界"}, + {"Hello, 世界", 1, "H"}, + {"Hello, 世界", 0, ""}, + {"Hello, 世界", 20, "Hello, 世界"}, + {"Hello, 世界", -20, "Hello, 世界"}, + } + for _, tt := range tests { + t.Run(tt.s, func(t *testing.T) { + if got := trunc(tt.c, tt.s); got != tt.want { + t.Errorf("trunc() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_substring(t *testing.T) { + + tests := []struct { + start int + end int + s string + want string + }{ + {1, 8, "Hello, 世界", "ello, 世"}, + {-10, 8, "Hello, 世界", "Hello, 世"}, + {1, 10, "Hello, 世界", "ello, 世界"}, + {-1, 10, "Hello, 世界", "Hello, 世界"}, + {-1, 1, "Hello, 世界", "H"}, + {-1, -1, "Hello, 世界", ""}, + {20, -1, "Hello, 世界", ""}, + {1, 1, "Hello, 世界", ""}, + {5, 1, "Hello, 世界", ""}, + {3, -1, "Hello, 世界", "lo, 世界"}, + } + for _, tt := range tests { + t.Run(tt.s, func(t *testing.T) { + if got := substring(tt.start, tt.end, tt.s); got != tt.want { + t.Errorf("substring() = %v, want %v", got, tt.want) + } + }) + } +} From 0fb3dcc1bddd4896474a07e492f0c90bd21a4ed3 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:32:13 +0100 Subject: [PATCH 28/39] Fixes panic out of boundary Signed-off-by: Cyril Tovena --- pkg/logql/log/fmt.go | 2 +- pkg/logql/log/fmt_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/logql/log/fmt.go b/pkg/logql/log/fmt.go index b9d794bce37cc..745aed711974c 100644 --- a/pkg/logql/log/fmt.go +++ b/pkg/logql/log/fmt.go @@ -189,7 +189,7 @@ func (lf *LabelsFormatter) Process(l []byte, lbs *LabelsBuilder) ([]byte, bool) func trunc(c int, s string) string { runes := []rune(s) - l := len(s) + l := len(runes) if c < 0 && l+c > 0 { return string(runes[l+c:]) } diff --git a/pkg/logql/log/fmt_test.go b/pkg/logql/log/fmt_test.go index 0668798d3edbb..b01ea54edf9eb 100644 --- a/pkg/logql/log/fmt_test.go +++ b/pkg/logql/log/fmt_test.go @@ -1,6 +1,7 @@ package log import ( + "fmt" "sort" "testing" @@ -254,7 +255,7 @@ func Test_trunc(t *testing.T) { {"Hello, 世界", -20, "Hello, 世界"}, } for _, tt := range tests { - t.Run(tt.s, func(t *testing.T) { + t.Run(fmt.Sprintf("%s%d", tt.s, tt.c), func(t *testing.T) { if got := trunc(tt.c, tt.s); got != tt.want { t.Errorf("trunc() = %v, want %v", got, tt.want) } From 0bfbc1ca4eb78e2056927cb1ad6a9f557d788901 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:58:08 -0500 Subject: [PATCH 29/39] Update docs/sources/logql/functions.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index fa72a0f2b6747..76a0f7186b14d 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -21,7 +21,7 @@ Example: ## ToLower & ToUpper -Convert the entire string to lowercase or uppercase. +This function converts the entire string to lowercase or uppercase. Signatures: From 4c18044d576a7d91c22023e09e7656c7fafeb9b2 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:58:20 -0500 Subject: [PATCH 30/39] Update docs/sources/logql/functions.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 76a0f7186b14d..8d8b6dfa396d6 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -217,7 +217,7 @@ Examples: > **Note:** Added in Loki 2.1. -Perform simple string replacement. +This function performs simple string replacement. Signature: `replace(old string, new string, src string) string` From 7bd59ecac191b7795d7dfe6790a4f57a5b0d02be Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:58:29 -0500 Subject: [PATCH 31/39] Update docs/sources/logql/functions.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 8d8b6dfa396d6..571d03f5ce771 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -255,7 +255,7 @@ Examples: > **Note:** Added in Loki 2.1. -Remove given characters from the front or back of a string. +Use this function to remove given characters from the front or back of a string. Signature: `trimAll(chars string,src string) string` From 84231afc3cf404053f965579898861d92ef0f58a Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:58:40 -0500 Subject: [PATCH 32/39] Update docs/sources/logql/functions.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 571d03f5ce771..35774b414cbdf 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -270,7 +270,7 @@ Examples: > **Note:** Added in Loki 2.1. -Trim just the suffix from a string. +Use this function to trim just the suffix from a string. Signature: `trimSuffix(suffix string, src string) string` From 257947eb73eb786558e8a49dbff897540f79b60f Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:59:18 -0500 Subject: [PATCH 33/39] Update docs/sources/logql/_index.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index d7e9beeb0b3b5..cd060263724cc 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -303,7 +303,7 @@ The `| label_format` expression can renamed, modify or add labels. It takes as p When both side are label identifiers, for example `dst=src`, the operation will rename the `src` label into `dst`. -The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value will be replace by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, which means labels are available as variables and you can use the same list of [functions](functions/). +The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value is replaced by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, which means labels are available as variables and you can use the same list of [functions](functions/). In both case if the destination label doesn't exist, then a new one is created. From 128ae1fbeeaed7ec2d1e070fd19983b11eb25368 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:59:34 -0500 Subject: [PATCH 34/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 35774b414cbdf..19f77dad9f3e2 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -4,7 +4,7 @@ title: Template functions The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support functions the usage of functions. -All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`) for example the following template will output the value of the path label: +All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`). For example the following template will output the value of the path label: ```template {{ .path }} From 641543fb4720c6b89dbf7824ddda7cf074739e5f Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:59:44 -0500 Subject: [PATCH 35/39] Update docs/sources/logql/functions.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 19f77dad9f3e2..859908c3f32eb 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -36,7 +36,7 @@ Examples: `{{ToUpper "This is a string" | ToLower}}` ``` -> **Note:** In Loki 2.1 you can also use respectively [`lower`](#lower) and [`upper`](#upper) shortcut, e.g `{{.request_method | lower }}` +> **Note:** In Loki 2.1 you can also use respectively [`lower`](#lower) and [`upper`](#upper) shortcut, e.g `{{.request_method | lower }}`. ## Replace string From 3be39f3da1a380ee244f4bcadb025066e36afe6f Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 14:59:53 -0500 Subject: [PATCH 36/39] Update docs/sources/logql/_index.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index cd060263724cc..e2a8427416a89 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -305,7 +305,7 @@ When both side are label identifiers, for example `dst=src`, the operation will The left side can alternatively be a template string (double quoted or backtick), for example `dst="{{.status}} {{.query}}"`, in which case the `dst` label value is replaced by the result of the [text/template](https://golang.org/pkg/text/template/) evaluation. This is the same template engine as the `| line_format` expression, which means labels are available as variables and you can use the same list of [functions](functions/). -In both case if the destination label doesn't exist, then a new one is created. +In both cases, if the destination label doesn't exist, then a new one is created. The renaming form `dst=src` will _drop_ the `src` label after remapping it to the `dst` label. However, the _template_ form will preserve the referenced labels, such that `dst="{{.src}}"` results in both `dst` and `src` having the same value. From a9a7579a8a08653120685c40c5f02c5d700c6d68 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 15:00:52 -0500 Subject: [PATCH 37/39] Apply suggestions from code review Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/logql/functions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 859908c3f32eb..7e7a1a84b72fc 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -2,7 +2,7 @@ title: Template functions --- -The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support functions the usage of functions. +The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support the usage of functions. All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`). For example the following template will output the value of the path label: @@ -10,7 +10,7 @@ All labels are added as variables in the template engine. They can be referenced {{ .path }} ``` -You can take advantage of [pipeline](https://golang.org/pkg/text/template/#hdr-Pipelines) to chain multiple functions. +You can take advantage of [pipeline](https://golang.org/pkg/text/template/#hdr-Pipelines) to join together multiple functions. In a chained pipeline, the result of each command is passed as the last argument of the following command. Example: @@ -19,7 +19,7 @@ Example: {{ .path | replace " " "_" | trunc 5 | upper }} ``` -## ToLower & ToUpper +## ToLower and ToUpper This function converts the entire string to lowercase or uppercase. @@ -42,7 +42,7 @@ Examples: > **Note:** In Loki 2.1 [`replace`](#replace) (as opposed to `Replace`) is available with a different signature but easier to chain within pipeline. -Perform simple string replacement. +Use this function to perform a simple string replacement. Signature: @@ -61,9 +61,9 @@ Example: `{{ Replace "This is a string" " " "-" -1 }}` ``` -The above will produce `This-is-a-string`. +The results in `This-is-a-string`. -## Trim, TrimLeft, TrimRight and TrimSpace +## Trim, TrimLeft, TrimRight, and TrimSpace > **Note:** In Loki 2.1 [trim](#trim), [trimAll](#trimAll), [trimSuffix](#trimSuffix) and [trimPrefix](trimPrefix) have been added with a different signature for better pipeline chaining. @@ -72,7 +72,7 @@ trailing Unicode code points contained in cutset removed. Signature: `Trim(value, cutset string) string` -`TrimLeft` and `TrimRight` are the same as `Trim` except that it respectively trim only leading and trailing characters. +`TrimLeft` and `TrimRight` are the same as `Trim` except that it trims only leading and trailing characters respectively. ```template `{{ Trim .query ",. " }}` @@ -102,7 +102,7 @@ Signature: ## regexReplaceAll and regexReplaceAllLiteral -`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [Regexp.replaceAll documentation](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for detailed examples. +`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [Regexp.replaceAll documentation](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for more examples. ```template `{{ regexReplaceAllLiteral "(a*)bc" .some_label "${1}a" }}` From aaee685be186056f4460a3f7b9db4c6c0ab59d41 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 15:01:48 -0500 Subject: [PATCH 38/39] Apply suggestions from code review Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/functions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 7e7a1a84b72fc..7c5a96c7d2c3d 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -108,19 +108,19 @@ Signature: `{{ regexReplaceAllLiteral "(a*)bc" .some_label "${1}a" }}` ``` -`regexReplaceAllLiteral` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement The replacement string is substituted directly, without using Expand. +`regexReplaceAllLiteral` function returns a copy of the input string and replaces matches of the Regexp with the replacement string replacement. The replacement string is substituted directly, without using Expand. ```template `{{ regexReplaceAllLiteral "(ts=)" .timestamp "timestamp=" }}` ``` -You can combine multiple function using pipe. For example, if you want to strip out spaces and make the request method in capital, then you would write the following template: `{{ .request_method | TrimSpace | ToUpper }}`. +You can combine multiple functions using pipe. For example, to strip out spaces and make the request method in capital, you would write the following template: `{{ .request_method | TrimSpace | ToUpper }}`. ## lower > Added in Loki 2.1 -Convert to lower case. +Use this function to convert to lower case. Signature: @@ -139,7 +139,7 @@ The last example will return `hello`. > Added in Loki 2.1 -Convert to upper case. +Use this function to convert to upper case. Signature: @@ -152,7 +152,7 @@ Examples: `{{ upper "hello"}}` ``` -The last example will return `HELLO`. +This results in `HELLO`. ## title @@ -285,7 +285,7 @@ Examples: > **Note:** Added in Loki 2.1. -Trim just the prefix from a string. +Use this function to trim just the prefix from a string. Signature: `trimPrefix(suffix string, src string) string` @@ -308,7 +308,7 @@ Signature: `indent(spaces int,src string) string` {{ indent 4 .query }} ``` -The following will indent by 4 spaces each lines contained in `.query`. +This indents each line contained in the `.query` by four (4) spaces. ## nindent @@ -322,7 +322,7 @@ Signature: `nindent(spaces int,src string) string` {{ nindent 4 .query }} ``` -The above will indent every line of text by 4 space characters and add a new line to the beginning. +This will indent every line of text by 4 space characters and add a new line to the beginning. ## repeat From c97fcf238d51148ffc03f3e02823a5d97af3b5cd Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 5 Nov 2020 15:02:17 -0500 Subject: [PATCH 39/39] Apply suggestions from code review Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- docs/sources/logql/functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/logql/functions.md b/docs/sources/logql/functions.md index 7c5a96c7d2c3d..1728aea06053a 100644 --- a/docs/sources/logql/functions.md +++ b/docs/sources/logql/functions.md @@ -328,7 +328,7 @@ This will indent every line of text by 4 space characters and add a new line to > **Note:** Added in Loki 2.1. -Repeat a string multiple times. +Use this function to repeat a string multiple times. Signature: `repeat(c int,value string) string` @@ -340,7 +340,7 @@ Signature: `repeat(c int,value string) string` > **Note:** Added in Loki 2.1. -Test to see if one string is contained inside of another. +Use this function to test to see if one string is contained inside of another. Signature: `contains(s string, src string) bool`