From 951d6f8e9baf1197c85c2e420068ad047db33c45 Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Mon, 12 Aug 2024 22:18:04 +0200 Subject: [PATCH] [pkg/ottl] add support for map literals (#34168) **Description:** This PR extends the OTTL grammar to support map literals **Link to tracking Issue:** #32388 **Testing:** Unit tests; E2E Tests **Documentation:** Extended the docs with a section about the added data type --------- Signed-off-by: Florian Bacher --- .chloggen/ottl-map-literals.yaml | 27 +++++ pkg/ottl/LANGUAGE.md | 11 +++ pkg/ottl/e2e/e2e_test.go | 19 ++++ pkg/ottl/expression.go | 38 +++++++ pkg/ottl/expression_test.go | 120 ++++++++++++++++++++++- pkg/ottl/grammar.go | 13 +++ pkg/ottl/lexer_test.go | 9 +- pkg/ottl/parser_test.go | 163 +++++++++++++++++++++++++++++++ 8 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 .chloggen/ottl-map-literals.yaml diff --git a/.chloggen/ottl-map-literals.yaml b/.chloggen/ottl-map-literals.yaml new file mode 100644 index 000000000000..87e9cac03772 --- /dev/null +++ b/.chloggen/ottl-map-literals.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for map literals in OTTL + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32388] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/LANGUAGE.md b/pkg/ottl/LANGUAGE.md index a1ac77a57dc4..c5e09d58ff7b 100644 --- a/pkg/ottl/LANGUAGE.md +++ b/pkg/ottl/LANGUAGE.md @@ -114,6 +114,7 @@ Values are passed as function parameters or are used in a Boolean Expression. Va - [Enums](#enums) - [Converters](#converters) - [Math Expressions](#math-expressions) +- [Maps](#maps) ### Paths @@ -155,6 +156,16 @@ Example List Values: - `["1", "2", "3"]` - `["a", attributes["key"], Concat(["a", "b"], "-")]` +### Maps + +A Map Value comprises a set of key Value pairs. + +Example Map Values: +- `{}` +- `{"foo": "bar"}` +- `{"foo": {"a": 2}}` +- `{"foo": {"a": attributes["key"]}}` + ### Literals Literals are literal interpretations of the Value into a Go value. Accepted literals are: diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index 1e72d314633d..7fed70409c4b 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -860,6 +860,25 @@ func Test_e2e_ottl_features(t *testing.T) { statement: `set(attributes["test"], attributes["metadata"]["uid"])`, want: func(_ ottllog.TransformContext) {}, }, + { + name: "map value", + statement: `set(body, {"_raw": body, "test": {"result": attributes["foo"]["bar"], "time": UnixNano(time)}})`, + want: func(tCtx ottllog.TransformContext) { + originalBody := tCtx.GetLogRecord().Body().AsString() + mapValue := tCtx.GetLogRecord().Body().SetEmptyMap() + mapValue.PutStr("_raw", originalBody) + mv1 := mapValue.PutEmptyMap("test") + mv1.PutStr("result", "pass") + mv1.PutInt("time", 1581452772000000321) + }, + }, + { + name: "map value as input to function", + statement: `set(attributes["isMap"], IsMap({"foo": {"bar": "baz", "test": "pass"}}))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutBool("isMap", true) + }, + }, } for _, tt := range tests { diff --git a/pkg/ottl/expression.go b/pkg/ottl/expression.go index c1cb833c66ce..ceae91386a84 100644 --- a/pkg/ottl/expression.go +++ b/pkg/ottl/expression.go @@ -143,6 +143,32 @@ func (l *listGetter[K]) Get(ctx context.Context, tCtx K) (any, error) { return evaluated, nil } +type mapGetter[K any] struct { + mapValues map[string]Getter[K] +} + +func (m *mapGetter[K]) Get(ctx context.Context, tCtx K) (any, error) { + evaluated := map[string]any{} + for k, v := range m.mapValues { + val, err := v.Get(ctx, tCtx) + if err != nil { + return nil, err + } + switch t := val.(type) { + case pcommon.Map: + evaluated[k] = t.AsRaw() + default: + evaluated[k] = t + } + + } + result := pcommon.NewMap() + if err := result.FromRaw(evaluated); err != nil { + return nil, err + } + return result, nil +} + // TypeError represents that a value was not an expected type. type TypeError string @@ -738,6 +764,18 @@ func (p *Parser[K]) newGetter(val value) (Getter[K], error) { return &lg, nil } + if val.Map != nil { + mg := mapGetter[K]{mapValues: map[string]Getter[K]{}} + for _, kvp := range val.Map.Values { + getter, err := p.newGetter(*kvp.Value) + if err != nil { + return nil, err + } + mg.mapValues[*kvp.Key] = getter + } + return &mg, nil + } + if val.MathExpression == nil { // In practice, can't happen since the DSL grammar guarantees one is set return nil, fmt.Errorf("no value field set. This is a bug in the OpenTelemetry Transformation Language") diff --git a/pkg/ottl/expression_test.go b/pkg/ottl/expression_test.go index 511d5d47b487..4aa426e558b3 100644 --- a/pkg/ottl/expression_test.go +++ b/pkg/ottl/expression_test.go @@ -414,6 +414,116 @@ func Test_newGetter(t *testing.T) { }, want: []any{"test0", int64(1)}, }, + { + name: "map", + val: value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("stringAttr"), + Value: &value{String: ottltest.Strp("value")}, + }, + { + Key: ottltest.Strp("intAttr"), + Value: &value{ + Literal: &mathExprLiteral{ + Int: ottltest.Intp(3), + }, + }, + }, + { + Key: ottltest.Strp("floatAttr"), + Value: &value{ + Literal: &mathExprLiteral{ + Float: ottltest.Floatp(2.5), + }, + }, + }, + { + Key: ottltest.Strp("boolAttr"), + Value: &value{Bool: (*boolean)(ottltest.Boolp(true))}, + }, + { + Key: ottltest.Strp("byteAttr"), + Value: &value{Bytes: (*byteSlice)(&[]byte{1, 2, 3, 4, 5, 6, 7, 8})}, + }, + { + Key: ottltest.Strp("enumAttr"), + Value: &value{Enum: (*enumSymbol)(ottltest.Strp("TEST_ENUM_ONE"))}, + }, + { + Key: ottltest.Strp("pathAttr"), + Value: &value{ + Literal: &mathExprLiteral{ + Path: &path{ + Fields: []field{ + { + Name: "name", + }, + }, + }, + }, + }, + }, + { + Key: ottltest.Strp("mapAttr"), + Value: &value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("foo"), + Value: &value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("test"), + Value: &value{String: ottltest.Strp("value")}, + }, + }, + }, + }, + }, + { + Key: ottltest.Strp("listAttr"), + Value: &value{ + List: &list{ + Values: []value{ + { + String: ottltest.Strp("test0"), + }, + { + Literal: &mathExprLiteral{ + Int: ottltest.Intp(1), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + ctx: "bear", + want: map[string]any{ + "enumAttr": int64(1), + "pathAttr": "bear", + "mapAttr": map[string]any{ + "foo": map[string]any{ + "test": "value", + }, + "listAttr": []any{"test0", int64(1)}, + }, + "stringAttr": "value", + "intAttr": int64(3), + "floatAttr": 2.5, + "boolAttr": true, + "byteAttr": []byte{1, 2, 3, 4, 5, 6, 7, 8}, + }, + }, } functions := CreateFactoryMap( @@ -444,7 +554,15 @@ func Test_newGetter(t *testing.T) { val, err := reader.Get(context.Background(), tCtx) assert.NoError(t, err) - assert.Equal(t, tt.want, val) + + switch v := val.(type) { + case pcommon.Map: + // need to compare the raw map here as require.EqualValues can not seem to handle + // the comparison of pcommon.Map + assert.EqualValues(t, tt.want, v.AsRaw()) + default: + assert.EqualValues(t, tt.want, v) + } }) } diff --git a/pkg/ottl/grammar.go b/pkg/ottl/grammar.go index 04352a6b7a8e..44f642b78504 100644 --- a/pkg/ottl/grammar.go +++ b/pkg/ottl/grammar.go @@ -237,6 +237,7 @@ type value struct { String *string `parser:"| @String"` Bool *boolean `parser:"| @Boolean"` Enum *enumSymbol `parser:"| @Uppercase (?! Lowercase)"` + Map *mapValue `parser:"| @@"` List *list `parser:"| @@)"` } @@ -270,6 +271,15 @@ type list struct { Values []value `parser:"'[' (@@)* (',' @@)* ']'"` } +type mapValue struct { + Values []mapItem `parser:"'{' (@@ ','?)* '}'"` +} + +type mapItem struct { + Key *string `parser:"@String ':'"` + Value *value `parser:"@@"` +} + // byteSlice type for capturing byte slices type byteSlice []byte @@ -444,6 +454,9 @@ func buildLexer() *lexer.StatefulDefinition { {Name: `Equal`, Pattern: `=`}, {Name: `LParen`, Pattern: `\(`}, {Name: `RParen`, Pattern: `\)`}, + {Name: `LBrace`, Pattern: `\{`}, + {Name: `RBrace`, Pattern: `\}`}, + {Name: `Colon`, Pattern: `\:`}, {Name: `Punct`, Pattern: `[,.\[\]]`}, {Name: `Uppercase`, Pattern: `[A-Z][A-Z0-9_]*`}, {Name: `Lowercase`, Pattern: `[a-z][a-z0-9_]*`}, diff --git a/pkg/ottl/lexer_test.go b/pkg/ottl/lexer_test.go index 289145f8a05c..b73d71d80e8f 100644 --- a/pkg/ottl/lexer_test.go +++ b/pkg/ottl/lexer_test.go @@ -78,7 +78,7 @@ func Test_lexer(t *testing.T) { {"OpNot", "not"}, {"Boolean", "false"}, }}, - {"nothing_recognizable", "{}", true, []result{ + {"nothing_recognizable", "|", true, []result{ {"", ""}, }}, {"basic_ident_expr", `set(attributes["bytes"], 0x0102030405060708)`, false, []result{ @@ -123,6 +123,13 @@ func Test_lexer(t *testing.T) { {"OpMultDiv", "*"}, {"Float", "2.9"}, }}, + {"Map", `{"foo":"bar"}`, false, []result{ + {"LBrace", "{"}, + {"String", `"foo"`}, + {"Colon", ":"}, + {"String", `"bar"`}, + {"RBrace", "}"}, + }}, } for _, tt := range tests { diff --git a/pkg/ottl/parser_test.go b/pkg/ottl/parser_test.go index 679ceddd1872..409d5ab34b5d 100644 --- a/pkg/ottl/parser_test.go +++ b/pkg/ottl/parser_test.go @@ -89,6 +89,169 @@ func Test_parse(t *testing.T) { WhereClause: nil, }, }, + { + name: "editor with map", + statement: `fff({"stringAttr": "value", "intAttr": 3, "floatAttr": 2.5, "boolAttr": true})`, + expected: &parsedStatement{ + Editor: editor{ + Function: "fff", + Arguments: []argument{ + { + Value: value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("stringAttr"), + Value: &value{String: ottltest.Strp("value")}, + }, + { + Key: ottltest.Strp("intAttr"), + Value: &value{ + Literal: &mathExprLiteral{ + Int: ottltest.Intp(3), + }, + }, + }, + { + Key: ottltest.Strp("floatAttr"), + Value: &value{ + Literal: &mathExprLiteral{ + Float: ottltest.Floatp(2.5), + }, + }, + }, + { + Key: ottltest.Strp("boolAttr"), + Value: &value{Bool: (*boolean)(ottltest.Boolp(true))}, + }, + }, + }, + }, + }, + }, + }, + WhereClause: nil, + }, + }, + { + name: "editor with empty map", + statement: `fff({})`, + expected: &parsedStatement{ + Editor: editor{ + Function: "fff", + Arguments: []argument{ + { + Value: value{ + Map: &mapValue{ + Values: nil, + }, + }, + }, + }, + }, + WhereClause: nil, + }, + }, + { + name: "editor with converter with a map", + statement: `fff(GetSomething({"foo":"bar"}))`, + expected: &parsedStatement{ + Editor: editor{ + Function: "fff", + Arguments: []argument{ + { + Value: value{ + Literal: &mathExprLiteral{ + Converter: &converter{ + Function: "GetSomething", + Arguments: []argument{ + { + Value: value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("foo"), + Value: &value{String: ottltest.Strp("bar")}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + WhereClause: nil, + }, + }, + { + name: "editor with nested map", + statement: `fff({"mapAttr": {"foo": "bar", "get": bear.honey, "arrayAttr":["foo", "bar"]}})`, + expected: &parsedStatement{ + Editor: editor{ + Function: "fff", + Arguments: []argument{ + { + Value: value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("mapAttr"), + Value: &value{ + Map: &mapValue{ + Values: []mapItem{ + { + Key: ottltest.Strp("foo"), + Value: &value{String: ottltest.Strp("bar")}, + }, + { + Key: ottltest.Strp("get"), + Value: &value{ + Literal: &mathExprLiteral{ + Path: &path{ + Fields: []field{ + { + Name: "bear", + }, + { + Name: "honey", + }, + }, + }, + }, + }, + }, + { + Key: ottltest.Strp("arrayAttr"), + Value: &value{ + List: &list{ + Values: []value{ + { + String: ottltest.Strp("foo"), + }, + { + String: ottltest.Strp("bar"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + WhereClause: nil, + }, + }, { name: "complex editor", statement: `set("foo", GetSomething(bear.honey))`,