From 4e953630efd396f833c2b5051eeaefb5c1456ea0 Mon Sep 17 00:00:00 2001 From: Christian Allinson Date: Mon, 25 Sep 2023 21:42:51 -0400 Subject: [PATCH 1/5] return Error when result is optional --- plugins/parsers/json_v2/parser.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/parsers/json_v2/parser.go b/plugins/parsers/json_v2/parser.go index bb538d1bf384d..53651f65e7808 100644 --- a/plugins/parsers/json_v2/parser.go +++ b/plugins/parsers/json_v2/parser.go @@ -703,9 +703,8 @@ func (p *Parser) convertType(input gjson.Result, desiredType string, name string func (p *Parser) checkResult(result gjson.Result, path string, optional bool) (bool, error) { if !result.Exists() { if optional { - // If path is marked as optional don't error if path doesn't return a result p.Log.Debugf("the path %s doesn't exist", path) - return true, nil + return true, fmt.Errorf("the path %s doesn't exist", path) } return false, fmt.Errorf("the path %s doesn't exist", path) From ce9f1af9827c8d93889c6a61f0eaef36aadbde7c Mon Sep 17 00:00:00 2001 From: Christian Allinson Date: Tue, 26 Sep 2023 16:51:01 -0400 Subject: [PATCH 2/5] Added tests --- .../nested_objects_optional/expected.out | 5 ++ .../nested_objects_nest.json | 25 ++++++++ .../nested_objects_single.json | 6 ++ .../nested_objects_optional/telegraf.conf | 58 +++++++++++++++++++ 4 files changed, 94 insertions(+) create mode 100644 plugins/parsers/json_v2/testdata/nested_objects_optional/expected.out create mode 100644 plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_nest.json create mode 100644 plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_single.json create mode 100644 plugins/parsers/json_v2/testdata/nested_objects_optional/telegraf.conf diff --git a/plugins/parsers/json_v2/testdata/nested_objects_optional/expected.out b/plugins/parsers/json_v2/testdata/nested_objects_optional/expected.out new file mode 100644 index 0000000000000..65b35d138038e --- /dev/null +++ b/plugins/parsers/json_v2/testdata/nested_objects_optional/expected.out @@ -0,0 +1,5 @@ +file,datatype=float,name=inlet-1/temperature value_f=0.20791169081775931 1695679023000000000 +file,datatype=float value_f=-0.3090169943749477,name="inlet-1/temperature" 1695679077000000000 +file,datatype=string value_s="-0.3090169943749477",name="inlet-3/flow" 1695679077000000000 +file,datatype=int value_i=95i,name="inlet-2/temperature" 1695679077000000000 +file,datatype=bool value_b=true,name="inlet-2/flow" 1695679077000000000 \ No newline at end of file diff --git a/plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_nest.json b/plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_nest.json new file mode 100644 index 0000000000000..709d596dcf42f --- /dev/null +++ b/plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_nest.json @@ -0,0 +1,25 @@ +{ + "timestamp": 1695679077118.0, + "metrics": [ + { + "value": -0.3090169943749477, + "name": "inlet-1/temperature", + "datatype": "float" + }, + { + "value": 95, + "name": "inlet-2/temperature", + "datatype": "int" + }, + { + "value": true, + "name": "inlet-2/flow", + "datatype": "bool" + }, + { + "value": "-0.3090169943749477", + "name": "inlet-3/flow", + "datatype": "string" + } + ] +} \ No newline at end of file diff --git a/plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_single.json b/plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_single.json new file mode 100644 index 0000000000000..6f54bf5ba5579 --- /dev/null +++ b/plugins/parsers/json_v2/testdata/nested_objects_optional/nested_objects_single.json @@ -0,0 +1,6 @@ +{ + "timestamp": 1695679022882.0, + "value": 0.20791169081775931, + "name": "inlet-1/temperature", + "datatype": "float" +} \ No newline at end of file diff --git a/plugins/parsers/json_v2/testdata/nested_objects_optional/telegraf.conf b/plugins/parsers/json_v2/testdata/nested_objects_optional/telegraf.conf new file mode 100644 index 0000000000000..78e6a9fa05d74 --- /dev/null +++ b/plugins/parsers/json_v2/testdata/nested_objects_optional/telegraf.conf @@ -0,0 +1,58 @@ +# Example taken from: https://github.com/influxdata/telegraf/issues/13990 + +# Parse String types from JSON +[[inputs.file]] + + files = ["./testdata/nested_objects_optional/nested_objects_single.json", "./testdata/nested_objects_optional/nested_objects_nest.json"] + data_format = "json_v2" + + [[inputs.file.json_v2]] + timestamp_path = 'timestamp' + timestamp_format = 'unix_ms' + + [[inputs.file.json_v2.tag]] + path = 'name' + optional = true + [[inputs.file.json_v2.tag]] + path = 'datatype' + optional = true + [[inputs.file.json_v2.field]] + path = 'value' + rename = 'value_f' + optional = true + + [[inputs.file.json_v2.object]] + path = 'metrics.#(datatype=="float")#' + optional = true + tags = ['datatype'] + [inputs.file.json_v2.object.renames] + value = 'value_f' + [inputs.file.json_v2.object.fields] + value = 'float' + + [[inputs.file.json_v2.object]] + path = 'metrics.#(datatype=="string")#' + optional = true + tags = ['datatype'] + [inputs.file.json_v2.object.renames] + value = 'value_s' + [inputs.file.json_v2.object.fields] + value = 'string' + + [[inputs.file.json_v2.object]] + path = 'metrics.#(datatype=="int")#' + optional = true + tags = ['datatype'] + [inputs.file.json_v2.object.renames] + value = 'value_i' + [inputs.file.json_v2.object.fields] + value = 'int' + + [[inputs.file.json_v2.object]] + path = 'metrics.#(datatype=="bool")#' + optional = true + tags = ['datatype'] + [inputs.file.json_v2.object.renames] + value = 'value_b' + [inputs.file.json_v2.object.fields] + value = 'bool' From 7fa75a30e4b93f29e68fd1e054452b8cf608eca9 Mon Sep 17 00:00:00 2001 From: Christian Allinson Date: Thu, 28 Sep 2023 08:40:16 -0400 Subject: [PATCH 3/5] Make relationship of arg optional and return value clear --- plugins/parsers/json_v2/parser.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/parsers/json_v2/parser.go b/plugins/parsers/json_v2/parser.go index dfde56891a225..cf630ebfde6de 100644 --- a/plugins/parsers/json_v2/parser.go +++ b/plugins/parsers/json_v2/parser.go @@ -700,14 +700,11 @@ func (p *Parser) convertType(input gjson.Result, desiredType string, name string return input.Value(), nil } +// Check if gjson result exists and if a non-existing gjson result can be skipped over func (p *Parser) checkResult(result gjson.Result, path string, optional bool) (bool, error) { if !result.Exists() { - if optional { - p.Log.Debugf("the path %q doesn't exist", path) - return true, fmt.Errorf("the path %q doesn't exist", path) - } - - return false, fmt.Errorf("the path %q doesn't exist", path) + p.Log.Debugf("the path %q doesn't exist", path) + return optional, fmt.Errorf("the path %q doesn't exist", path) } return false, nil From e8a3a10c74956f5cb8d9fb8616a3f10d45f207d5 Mon Sep 17 00:00:00 2001 From: Christian Allinson Date: Fri, 29 Sep 2023 16:36:21 -0400 Subject: [PATCH 4/5] Flatten checkResult to return only error --- plugins/parsers/json_v2/parser.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/parsers/json_v2/parser.go b/plugins/parsers/json_v2/parser.go index cf630ebfde6de..4fa9d5aad73db 100644 --- a/plugins/parsers/json_v2/parser.go +++ b/plugins/parsers/json_v2/parser.go @@ -210,8 +210,8 @@ func (p *Parser) processMetric(input []byte, data []DataSet, tag bool, timestamp return nil, fmt.Errorf("GJSON path is required") } result := gjson.GetBytes(input, c.Path) - if skip, err := p.checkResult(result, c.Path, c.Optional); err != nil { - if skip { + if err := p.checkResult(result, c.Path); err != nil { + if c.Optional { continue } return nil, err @@ -456,8 +456,8 @@ func (p *Parser) processObjects(input []byte, objects []Object, timestamp time.T } result := gjson.GetBytes(input, c.Path) - if skip, err := p.checkResult(result, c.Path, c.Optional); err != nil { - if skip { + if err := p.checkResult(result, c.Path); err != nil { + if c.Optional { continue } return nil, err @@ -467,8 +467,8 @@ func (p *Parser) processObjects(input []byte, objects []Object, timestamp time.T for _, f := range c.FieldPaths { var r PathResult r.result = gjson.GetBytes(scopedJSON, f.Path) - if skip, err := p.checkResult(r.result, f.Path, f.Optional); err != nil { - if skip { + if err := p.checkResult(r.result, f.Path); err != nil { + if f.Optional { continue } return nil, err @@ -480,8 +480,8 @@ func (p *Parser) processObjects(input []byte, objects []Object, timestamp time.T for _, f := range c.TagPaths { var r PathResult r.result = gjson.GetBytes(scopedJSON, f.Path) - if skip, err := p.checkResult(r.result, f.Path, f.Optional); err != nil { - if skip { + if err := p.checkResult(r.result, f.Path); err != nil { + if f.Optional { continue } return nil, err @@ -700,14 +700,14 @@ func (p *Parser) convertType(input gjson.Result, desiredType string, name string return input.Value(), nil } -// Check if gjson result exists and if a non-existing gjson result can be skipped over -func (p *Parser) checkResult(result gjson.Result, path string, optional bool) (bool, error) { +// Check if gjson result exists and return error if it does not +func (p *Parser) checkResult(result gjson.Result, path string) (error) { if !result.Exists() { p.Log.Debugf("the path %q doesn't exist", path) - return optional, fmt.Errorf("the path %q doesn't exist", path) + return fmt.Errorf("the path %q doesn't exist", path) } - return false, nil + return nil } func init() { From 7151d38c6f6c852ac3de4f879c111c050bdb487d Mon Sep 17 00:00:00 2001 From: Christian Allinson Date: Fri, 29 Sep 2023 20:58:58 +0000 Subject: [PATCH 5/5] Format fix --- plugins/parsers/json_v2/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/parsers/json_v2/parser.go b/plugins/parsers/json_v2/parser.go index 4fa9d5aad73db..c195cf1036e71 100644 --- a/plugins/parsers/json_v2/parser.go +++ b/plugins/parsers/json_v2/parser.go @@ -701,7 +701,7 @@ func (p *Parser) convertType(input gjson.Result, desiredType string, name string } // Check if gjson result exists and return error if it does not -func (p *Parser) checkResult(result gjson.Result, path string) (error) { +func (p *Parser) checkResult(result gjson.Result, path string) error { if !result.Exists() { p.Log.Debugf("the path %q doesn't exist", path) return fmt.Errorf("the path %q doesn't exist", path)