From 02fed0209647b35ed4e144c9ae269be3ac86fdce Mon Sep 17 00:00:00 2001 From: Dominik Rosiek <58699848+sumo-drosiek@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:42:15 +0200 Subject: [PATCH] [sumologicexporter]: add flatten_body to json logs (#295) * feat(vagrant): get builder version from makefile * feat(otelcolbuilder): use fluentforwardreceiver with complex log support * feat(sumologicexporter): add merge_body option to json logs --- otelcolbuilder/.otelcol-builder.yaml | 4 ++ pkg/exporter/sumologicexporter/README.md | 4 ++ pkg/exporter/sumologicexporter/config.go | 6 ++ pkg/exporter/sumologicexporter/factory.go | 1 + pkg/exporter/sumologicexporter/sender.go | 10 ++- pkg/exporter/sumologicexporter/sender_test.go | 71 ++++++++++++++++++- vagrant/provision.sh | 2 +- 7 files changed, 95 insertions(+), 3 deletions(-) diff --git a/otelcolbuilder/.otelcol-builder.yaml b/otelcolbuilder/.otelcol-builder.yaml index 27ffdbd350..02c4ccef9d 100644 --- a/otelcolbuilder/.otelcol-builder.yaml +++ b/otelcolbuilder/.otelcol-builder.yaml @@ -102,3 +102,7 @@ replaces: # Use features from branch https://github.com/sumologic/opentelemetry-collector-contrib/tree/v0.36.0-feat-routingprocessor-route-on-resource-attributes # This replacement should be removed when all the features from the branch are upstreamed. - github.com/open-telemetry/opentelemetry-collector-contrib/processor/routingprocessor => github.com/SumoLogic/opentelemetry-collector-contrib/processor/routingprocessor e4d0740a3729724df5b4864d43f4931185e41d47 + + # TODO: remove this when fluentforwardreceiver support for complex object is accepted, merged and released upstream + # PR: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/5676 + - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver => github.com/SumoLogic/opentelemetry-collector-contrib/receiver/fluentforwardreceiver 975436eaca3e805571a96e3712530c4ff63a349d diff --git a/pkg/exporter/sumologicexporter/README.md b/pkg/exporter/sumologicexporter/README.md index 642eb7e867..b18d20d792 100644 --- a/pkg/exporter/sumologicexporter/README.md +++ b/pkg/exporter/sumologicexporter/README.md @@ -73,6 +73,10 @@ exporters: # of the timestamp key. # default = "timestamp". timestamp_key: + # When flatten_body is set to true and log is a map, + # log's body is going to be flattened and `log_key` won't be used + # default = false + flatten_body: {true, false} # translate_attributes specifies whether attributes should be translated # from OpenTelemetry to Sumo conventions; diff --git a/pkg/exporter/sumologicexporter/config.go b/pkg/exporter/sumologicexporter/config.go index 3653621a08..311f6a4d45 100644 --- a/pkg/exporter/sumologicexporter/config.go +++ b/pkg/exporter/sumologicexporter/config.go @@ -107,6 +107,10 @@ type JSONLogs struct { // of the timestamp key. // By default this is "timestamp". TimestampKey string `mapstructure:"timestamp_key"` + // When flatten_body is set to true and log is a map, + // log's body is going to be flattened and `log_key` won't be used + // By default this is false. + FlattenBody bool `mapstructure:"flatten_body"` } // CreateDefaultHTTPClientSettings returns default http client settings @@ -197,4 +201,6 @@ const ( DefaultAddTimestamp bool = true // DefaultTimestampKey defines default TimestampKey value DefaultTimestampKey string = "timestamp" + // DefaultFlattenBody defines default FlattenBody value + DefaultFlattenBody bool = false ) diff --git a/pkg/exporter/sumologicexporter/factory.go b/pkg/exporter/sumologicexporter/factory.go index a06de352fe..0a3af46b3b 100644 --- a/pkg/exporter/sumologicexporter/factory.go +++ b/pkg/exporter/sumologicexporter/factory.go @@ -61,6 +61,7 @@ func createDefaultConfig() config.Exporter { LogKey: DefaultLogKey, AddTimestamp: DefaultAddTimestamp, TimestampKey: DefaultTimestampKey, + FlattenBody: DefaultFlattenBody, }, GraphiteTemplate: DefaultGraphiteTemplate, TraceFormat: OTLPTraceFormat, diff --git a/pkg/exporter/sumologicexporter/sender.go b/pkg/exporter/sumologicexporter/sender.go index c0856b4d01..4b01469c13 100644 --- a/pkg/exporter/sumologicexporter/sender.go +++ b/pkg/exporter/sumologicexporter/sender.go @@ -200,7 +200,15 @@ func (s *sender) logToJSON(record logPair) (string, error) { // Only append the body when it's not empty to prevent sending 'null' log. if body := record.log.Body(); !isEmptyAttributeValue(body) { - data.orig.Upsert(s.jsonLogsConfig.LogKey, body) + if s.jsonLogsConfig.FlattenBody && body.Type() == pdata.AttributeValueTypeMap { + // Cannot use CopyTo, as it overrides data.orig's values + body.MapVal().Range(func(k string, v pdata.AttributeValue) bool { + data.orig.Insert(k, v) + return true + }) + } else { + data.orig.Upsert(s.jsonLogsConfig.LogKey, body) + } } nextLine, err := json.Marshal(data.orig.AsRaw()) diff --git a/pkg/exporter/sumologicexporter/sender_test.go b/pkg/exporter/sumologicexporter/sender_test.go index 28ef0ef7b1..3fbd16a0d8 100644 --- a/pkg/exporter/sumologicexporter/sender_test.go +++ b/pkg/exporter/sumologicexporter/sender_test.go @@ -187,6 +187,36 @@ func exampleTwoLogs() []pdata.LogRecord { return buffer } +func exampleLogWithComplexBody() []pdata.LogRecord { + buffer := make([]pdata.LogRecord, 1) + buffer[0] = pdata.NewLogRecord() + body := pdata.NewAttributeValueMap().MapVal() + body.InsertString("a", "b") + body.InsertBool("c", false) + body.InsertInt("d", 20) + body.InsertDouble("e", 20.5) + + f := pdata.NewAttributeValueArray() + f.ArrayVal().EnsureCapacity(4) + f.ArrayVal().AppendEmpty().SetStringVal("p") + f.ArrayVal().AppendEmpty().SetBoolVal(true) + f.ArrayVal().AppendEmpty().SetIntVal(13) + f.ArrayVal().AppendEmpty().SetDoubleVal(19.3) + body.Insert("f", f) + + g := pdata.NewAttributeValueMap() + g.MapVal().InsertString("h", "i") + g.MapVal().InsertBool("j", false) + g.MapVal().InsertInt("k", 12) + g.MapVal().InsertDouble("l", 11.1) + + body.Insert("g", g) + + buffer[0].Attributes().InsertString("m", "n") + buffer[0].Body().SetMapVal(body) + return buffer +} + func exampleTwoDifferentLogs() []pdata.LogRecord { buffer := make([]pdata.LogRecord, 2) buffer[0] = pdata.NewLogRecord() @@ -403,6 +433,7 @@ func TestSendLogsJsonConfig(t *testing.T) { name string configOpts []func(*Config) bodyRegex string + logBuffer []logPair }{ { name: "default config", @@ -412,12 +443,14 @@ func TestSendLogsJsonConfig(t *testing.T) { LogKey: DefaultLogKey, AddTimestamp: DefaultAddTimestamp, TimestampKey: DefaultTimestampKey, + FlattenBody: DefaultFlattenBody, } }, }, bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + `\n` + `{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}`, + logBuffer: logRecordsToLogPair(exampleTwoLogs()), }, { name: "disabled add timestamp", @@ -432,6 +465,7 @@ func TestSendLogsJsonConfig(t *testing.T) { bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log"}` + `\n` + `{"key1":"value1","key2":"value2","log":"Another example log"}`, + logBuffer: logRecordsToLogPair(exampleTwoLogs()), }, { name: "enabled add timestamp with custom timestamp key", @@ -447,6 +481,7 @@ func TestSendLogsJsonConfig(t *testing.T) { bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log","xxyy_zz":\d{13}}` + `\n` + `{"key1":"value1","key2":"value2","log":"Another example log","xxyy_zz":\d{13}}`, + logBuffer: logRecordsToLogPair(exampleTwoLogs()), }, { name: "custom log key", @@ -456,12 +491,46 @@ func TestSendLogsJsonConfig(t *testing.T) { LogKey: "log_vendor_key", AddTimestamp: DefaultAddTimestamp, TimestampKey: DefaultTimestampKey, + FlattenBody: DefaultFlattenBody, } }, }, bodyRegex: `{"key1":"value1","key2":"value2","log_vendor_key":"Example log","timestamp":\d{13}}` + `\n` + `{"key1":"value1","key2":"value2","log_vendor_key":"Another example log","timestamp":\d{13}}`, + logBuffer: logRecordsToLogPair(exampleTwoLogs()), + }, + { + name: "flatten body", + configOpts: []func(*Config){ + func(c *Config) { + c.JSONLogs = JSONLogs{ + LogKey: "log_vendor_key", + AddTimestamp: DefaultAddTimestamp, + TimestampKey: DefaultTimestampKey, + FlattenBody: true, + } + }, + }, + bodyRegex: `{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` + + `"g":{"h":"i","j":false,"k":12,"l":11.1},"m":"n","timestamp":\d{13}}`, + logBuffer: logRecordsToLogPair(exampleLogWithComplexBody()), + }, + { + name: "complex body", + configOpts: []func(*Config){ + func(c *Config) { + c.JSONLogs = JSONLogs{ + LogKey: "log_vendor_key", + AddTimestamp: DefaultAddTimestamp, + TimestampKey: DefaultTimestampKey, + FlattenBody: DefaultFlattenBody, + } + }, + }, + bodyRegex: `{"log_vendor_key":{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` + + `"g":{"h":"i","j":false,"k":12,"l":11.1}},"m":"n","timestamp":\d{13}}`, + logBuffer: logRecordsToLogPair(exampleLogWithComplexBody()), }, } @@ -475,7 +544,7 @@ func TestSendLogsJsonConfig(t *testing.T) { }, tc.configOpts...) test.s.config.LogFormat = JSONFormat - test.s.logBuffer = logRecordsToLogPair(exampleTwoLogs()) + test.s.logBuffer = tc.logBuffer _, err := test.s.sendLogs(context.Background(), newFields(pdata.NewAttributeMap())) assert.NoError(t, err) diff --git a/vagrant/provision.sh b/vagrant/provision.sh index 78bc75cc25..a1eaed5d2f 100644 --- a/vagrant/provision.sh +++ b/vagrant/provision.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export BUILDER_VERSION=0.35.0 +export BUILDER_VERSION="$(grep '^BUILDER_VERSION' /sumologic/otelcolbuilder/Makefile | sed 's/BUILDER_VERSION ?= //')" export GO_VERSION=1.17 # Install opentelemetry-collector-builder