[sumologicexporter]: add flatten_body to json logs

otelcolbuilder/.otelcol-builder.yaml

@@ -102,3 +102,7 @@ replaces:
   # Use features from branch https://github.com/sumologic/opentelemetry-collector-contrib/tree/v0.36.0-feat-routingprocessor-route-on-resource-attributes
   # This replacement should be removed when all the features from the branch are upstreamed.
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/routingprocessor => github.com/SumoLogic/opentelemetry-collector-contrib/processor/routingprocessor e4d0740a3729724df5b4864d43f4931185e41d47
+
+  # TODO: remove this when fluentforwardreceiver support for complex object is accepted, merged and released upstream
+  # PR: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/5676
+  - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver => github.com/SumoLogic/opentelemetry-collector-contrib/receiver/fluentforwardreceiver 975436eaca3e805571a96e3712530c4ff63a349d

pkg/exporter/sumologicexporter/README.md

@@ -73,6 +73,10 @@ exporters:
       # of the timestamp key.
       # default = "timestamp".
       timestamp_key: <timestamp_key>
+      # When flatten_body is set to true and log is a map,
+      # log's body is going to be flatten and `log_key` won't be used
+      # default = false
+      flatten_body: {true, false}
 
     # translate_attributes specifies whether attributes should be translated
     # from OpenTelemetry to Sumo conventions;

pkg/exporter/sumologicexporter/config.go

@@ -107,6 +107,10 @@ type JSONLogs struct {
 	// of the timestamp key.
 	// By default this is "timestamp".
 	TimestampKey string `mapstructure:"timestamp_key"`
+	// When flatten_body is set to true and log is a map,
+	// log's body is going to be flatten and `log_key` won't be used
+	// By default this is false.
+	FlattenBody bool `mapstructure:"flatten_body"`
 }
 
 // CreateDefaultHTTPClientSettings returns default http client settings
@@ -197,4 +201,6 @@ const (
 	DefaultAddTimestamp bool = true
 	// DefaultTimestampKey defines default TimestampKey value
 	DefaultTimestampKey string = "timestamp"
+	// DefaultFlattenBody defines default FlattenBody value
+	DefaultFlattenBody bool = false
 )

pkg/exporter/sumologicexporter/factory.go

@@ -61,6 +61,7 @@ func createDefaultConfig() config.Exporter {
 			LogKey:       DefaultLogKey,
 			AddTimestamp: DefaultAddTimestamp,
 			TimestampKey: DefaultTimestampKey,
+			FlattenBody:  DefaultFlattenBody,
 		},
 		GraphiteTemplate: DefaultGraphiteTemplate,
 		TraceFormat:      OTLPTraceFormat,

pkg/exporter/sumologicexporter/sender.go

@@ -200,7 +200,15 @@ func (s *sender) logToJSON(record logPair) (string, error) {
 
 	// Only append the body when it's not empty to prevent sending 'null' log.
 	if body := record.log.Body(); !isEmptyAttributeValue(body) {
-		data.orig.Upsert(s.jsonLogsConfig.LogKey, body)
+		if s.jsonLogsConfig.FlattenBody && body.Type() == pdata.AttributeValueTypeMap {
+			// Cannot use CopyTo, as it overrides data.orig's values
+			body.MapVal().Range(func(k string, v pdata.AttributeValue) bool {
+				data.orig.Insert(k, v)
+				return true
+			})
+		} else {
+			data.orig.Upsert(s.jsonLogsConfig.LogKey, body)
+		}
 	}
 
 	nextLine, err := json.Marshal(data.orig.AsRaw())

pkg/exporter/sumologicexporter/sender_test.go

@@ -187,6 +187,36 @@ func exampleTwoLogs() []pdata.LogRecord {
 	return buffer
 }
 
+func exampleLogWithComplexBody() []pdata.LogRecord {
+	buffer := make([]pdata.LogRecord, 1)
+	buffer[0] = pdata.NewLogRecord()
+	body := pdata.NewAttributeValueMap().MapVal()
+	body.InsertString("a", "b")
+	body.InsertBool("c", false)
+	body.InsertInt("d", 20)
+	body.InsertDouble("e", 20.5)
+
+	f := pdata.NewAttributeValueArray()
+	f.ArrayVal().EnsureCapacity(4)
+	f.ArrayVal().AppendEmpty().SetStringVal("p")
+	f.ArrayVal().AppendEmpty().SetBoolVal(true)
+	f.ArrayVal().AppendEmpty().SetIntVal(13)
+	f.ArrayVal().AppendEmpty().SetDoubleVal(19.3)
+	body.Insert("f", f)
+
+	g := pdata.NewAttributeValueMap()
+	g.MapVal().InsertString("h", "i")
+	g.MapVal().InsertBool("j", false)
+	g.MapVal().InsertInt("k", 12)
+	g.MapVal().InsertDouble("l", 11.1)
+
+	body.Insert("g", g)
+
+	buffer[0].Attributes().InsertString("m", "n")
+	buffer[0].Body().SetMapVal(body)
+	return buffer
+}
+
 func exampleTwoDifferentLogs() []pdata.LogRecord {
 	buffer := make([]pdata.LogRecord, 2)
 	buffer[0] = pdata.NewLogRecord()
@@ -403,6 +433,7 @@ func TestSendLogsJsonConfig(t *testing.T) {
 		name       string
 		configOpts []func(*Config)
 		bodyRegex  string
+		logBuffer  []logPair
 	}{
 		{
 			name: "default config",
@@ -412,12 +443,14 @@ func TestSendLogsJsonConfig(t *testing.T) {
 						LogKey:       DefaultLogKey,
 						AddTimestamp: DefaultAddTimestamp,
 						TimestampKey: DefaultTimestampKey,
+						FlattenBody:  DefaultFlattenBody,
 					}
 				},
 			},
 			bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` +
 				`\n` +
 				`{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}`,
+			logBuffer: logRecordsToLogPair(exampleTwoLogs()),
 		},
 		{
 			name: "disabled add timestamp",
@@ -432,6 +465,7 @@ func TestSendLogsJsonConfig(t *testing.T) {
 			bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log"}` +
 				`\n` +
 				`{"key1":"value1","key2":"value2","log":"Another example log"}`,
+			logBuffer: logRecordsToLogPair(exampleTwoLogs()),
 		},
 		{
 			name: "enabled add timestamp with custom timestamp key",
@@ -447,6 +481,7 @@ func TestSendLogsJsonConfig(t *testing.T) {
 			bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log","xxyy_zz":\d{13}}` +
 				`\n` +
 				`{"key1":"value1","key2":"value2","log":"Another example log","xxyy_zz":\d{13}}`,
+			logBuffer: logRecordsToLogPair(exampleTwoLogs()),
 		},
 		{
 			name: "custom log key",
@@ -456,12 +491,46 @@ func TestSendLogsJsonConfig(t *testing.T) {
 						LogKey:       "log_vendor_key",
 						AddTimestamp: DefaultAddTimestamp,
 						TimestampKey: DefaultTimestampKey,
+						FlattenBody:  DefaultFlattenBody,
 					}
 				},
 			},
 			bodyRegex: `{"key1":"value1","key2":"value2","log_vendor_key":"Example log","timestamp":\d{13}}` +
 				`\n` +
 				`{"key1":"value1","key2":"value2","log_vendor_key":"Another example log","timestamp":\d{13}}`,
+			logBuffer: logRecordsToLogPair(exampleTwoLogs()),
+		},
+		{
+			name: "flatten body",
+			configOpts: []func(*Config){
+				func(c *Config) {
+					c.JSONLogs = JSONLogs{
+						LogKey:       "log_vendor_key",
+						AddTimestamp: DefaultAddTimestamp,
+						TimestampKey: DefaultTimestampKey,
+						FlattenBody:  true,
+					}
+				},
+			},
+			bodyRegex: `{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` +
+				`"g":{"h":"i","j":false,"k":12,"l":11.1},"m":"n","timestamp":\d{13}}`,
+			logBuffer: logRecordsToLogPair(exampleLogWithComplexBody()),
+		},
+		{
+			name: "complex body",
+			configOpts: []func(*Config){
+				func(c *Config) {
+					c.JSONLogs = JSONLogs{
+						LogKey:       "log_vendor_key",
+						AddTimestamp: DefaultAddTimestamp,
+						TimestampKey: DefaultTimestampKey,
+						FlattenBody:  DefaultFlattenBody,
+					}
+				},
+			},
+			bodyRegex: `{"log_vendor_key":{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` +
+				`"g":{"h":"i","j":false,"k":12,"l":11.1}},"m":"n","timestamp":\d{13}}`,
+			logBuffer: logRecordsToLogPair(exampleLogWithComplexBody()),
 		},
 	}
 
@@ -475,7 +544,7 @@ func TestSendLogsJsonConfig(t *testing.T) {
 			}, tc.configOpts...)
 
 			test.s.config.LogFormat = JSONFormat
-			test.s.logBuffer = logRecordsToLogPair(exampleTwoLogs())
+			test.s.logBuffer = tc.logBuffer
 
 			_, err := test.s.sendLogs(context.Background(), newFields(pdata.NewAttributeMap()))
 			assert.NoError(t, err)

vagrant/provision.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export BUILDER_VERSION=0.35.0
+export BUILDER_VERSION="$(grep '^BUILDER_VERSION' /sumologic/otelcolbuilder/Makefile | sed 's/BUILDER_VERSION ?= //')"
 export GO_VERSION=1.17
 
 # Install opentelemetry-collector-builder