[exporter/datadog]: add max tag length (#3185)

This PR is part of work to address Payload size errors reported here: #2676

Combined with batching of spans/traces, spans with very large attribute values such as stacktraces can cause payload sizes to become excessive. This PR adds a max tag (a datadog span attribute, basically), value helper function, and truncates tag values when they're too  large, which brings this in line with Datadog-Agent limits: https://github.com/DataDog/datadog-agent/blob/140a4ee164261ef2245340c50371ba989fbeb038/pkg/trace/traceutil/truncate.go#L23

Additionally, worth noting that work being done to directly import and re-use portions of the datadog-agent codebase will make this sort of thing much easier, and work being done within datadog to permit shorter batching times should also help resolve the Payload size errors being seen. However for now this PR can help.

**Link to tracking Issue:** <Issue number if applicable> 

#2676

exporter/datadogexporter/translate_traces.go

@@ -51,6 +51,9 @@ const (
 	eventNameTag        string = "name"
 	eventAttrTag        string = "attributes"
 	eventTimeTag        string = "time"
+	// max meta value from
+	// https://github.com/DataDog/datadog-agent/blob/140a4ee164261ef2245340c50371ba989fbeb038/pkg/trace/traceutil/truncate.go#L23.
+	MaxMetaValLen int = 5000
 	// tagContainersTags specifies the name of the tag which holds key/value
 	// pairs representing information about the container (Docker, EC2, etc).
 	tagContainersTags = "_dd.tags.container"
@@ -369,6 +372,10 @@ func setMetric(s *pb.Span, key string, v float64) {
 }
 
 func setStringTag(s *pb.Span, key, v string) {
+	if len(v) > MaxMetaValLen {
+		v = utils.TruncateUTF8(v, MaxMetaValLen)
+	}
+
 	switch key {
 	// if a span has `service.name` set as the tag
 	case ext.ServiceName:

exporter/datadogexporter/translate_traces_test.go

@@ -17,6 +17,7 @@ package datadogexporter
 import (
 	"bytes"
 	"fmt"
+	"math/rand"
 	"strings"
 	"testing"
 	"time"
@@ -37,6 +38,16 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/utils"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+func RandStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}
+
 func NewResourceSpansData(mockTraceID [16]byte, mockSpanID [8]byte, mockParentSpanID [8]byte, statusCode pdata.StatusCode, resourceEnvAndService bool, endTime time.Time) pdata.ResourceSpans {
 	// The goal of this test is to ensure that each span in
 	// pdata.ResourceSpans is transformed to its *trace.SpanData correctly!
@@ -718,6 +729,81 @@ func TestTracesTranslationServicePeerName(t *testing.T) {
 	assert.Equal(t, mockEventsString, datadogPayload.Traces[0].Spans[0].Meta["events"])
 }
 
+// ensure that the datadog span uses the truncated tags if length exceeds max
+func TestTracesTranslationTruncatetag(t *testing.T) {
+	hostname := "testhostname"
+	calculator := newSublayerCalculator()
+
+	// generate mock trace, span and parent span ids
+	mockTraceID := [16]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}
+	mockSpanID := [8]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8}
+	mockParentSpanID := [8]byte{0xEF, 0xEE, 0xED, 0xEC, 0xEB, 0xEA, 0xE9, 0xE8}
+	mockEndTime := time.Now().Round(time.Second)
+
+	// create mock resource span data
+	// set shouldError and resourceServiceandEnv to false to test defaut behavior
+	rs := NewResourceSpansData(mockTraceID, mockSpanID, mockParentSpanID, pdata.StatusCodeUnset, false, mockEndTime)
+
+	span := rs.InstrumentationLibrarySpans().At(0).Spans().At(0)
+
+	span.Attributes().InsertString(conventions.AttributeExceptionStacktrace, RandStringBytes(5500))
+
+	// translate mocks to datadog traces
+	datadogPayload := resourceSpansToDatadogSpans(rs, calculator, hostname, &config.Config{})
+	// ensure we return the correct type
+	assert.IsType(t, pb.TracePayload{}, datadogPayload)
+
+	// ensure hostname arg is respected
+	assert.Equal(t, hostname, datadogPayload.HostName)
+	assert.Equal(t, 1, len(datadogPayload.Traces))
+
+	// ensure trace id gets translated to uint64 correctly
+	assert.Equal(t, decodeAPMTraceID(mockTraceID), datadogPayload.Traces[0].TraceID)
+
+	// ensure the correct number of spans are expected
+	assert.Equal(t, 1, len(datadogPayload.Traces[0].Spans))
+
+	// ensure span's trace id matches payload trace id
+	assert.Equal(t, datadogPayload.Traces[0].TraceID, datadogPayload.Traces[0].Spans[0].TraceID)
+
+	// ensure span's spanId and parentSpanId are set correctly
+	assert.Equal(t, decodeAPMSpanID(mockSpanID), datadogPayload.Traces[0].Spans[0].SpanID)
+	assert.Equal(t, decodeAPMSpanID(mockParentSpanID), datadogPayload.Traces[0].Spans[0].ParentID)
+
+	// ensure that span.resource defaults to otlp span.name
+	assert.Equal(t, "End-To-End Here", datadogPayload.Traces[0].Spans[0].Resource)
+
+	// ensure that span.name defaults to string representing instrumentation library if present
+	assert.Equal(t, strings.ToLower(fmt.Sprintf("%s.%s", datadogPayload.Traces[0].Spans[0].Meta[conventions.InstrumentationLibraryName], strings.TrimPrefix(pdata.SpanKindSERVER.String(), "SPAN_KIND_"))), datadogPayload.Traces[0].Spans[0].Name)
+
+	// ensure that span.type is based on otlp span.kind
+	assert.Equal(t, "web", datadogPayload.Traces[0].Spans[0].Type)
+
+	// ensure that span.meta and span.metrics pick up attibutes, instrumentation ibrary and resource attribs
+	assert.Equal(t, 11, len(datadogPayload.Traces[0].Spans[0].Meta))
+	assert.Equal(t, 1, len(datadogPayload.Traces[0].Spans[0].Metrics))
+
+	// ensure that span error is based on otlp span status
+	assert.Equal(t, int32(0), datadogPayload.Traces[0].Spans[0].Error)
+
+	// ensure that span meta also inccludes correctly sets resource attributes
+	assert.Equal(t, "kube-system", datadogPayload.Traces[0].Spans[0].Meta["namespace"])
+
+	// ensure that span service name gives resource service.name priority
+	assert.Equal(t, 5000, len(datadogPayload.Traces[0].Spans[0].Meta[conventions.AttributeExceptionStacktrace]))
+
+	// ensure a duration and start time are calculated
+	assert.NotNil(t, datadogPayload.Traces[0].Spans[0].Start)
+	assert.NotNil(t, datadogPayload.Traces[0].Spans[0].Duration)
+
+	pdataMockEndTime := pdata.TimestampFromTime(mockEndTime)
+	pdataMockStartTime := pdata.TimestampFromTime(mockEndTime.Add(-90 * time.Second))
+	mockEventsString := fmt.Sprintf("[{\"attributes\":{},\"name\":\"start\",\"time\":%d},{\"attributes\":{\"flag\":false},\"name\":\"end\",\"time\":%d}]", pdataMockStartTime, pdataMockEndTime)
+
+	// ensure that events tag is set if span events exist and contains structured json fields
+	assert.Equal(t, mockEventsString, datadogPayload.Traces[0].Spans[0].Meta["events"])
+}
+
 // ensure that datadog span resource naming uses http method+route when available
 func TestSpanResourceTranslation(t *testing.T) {
 	span := pdata.NewSpan()

exporter/datadogexporter/utils/trace_helpers.go

@@ -134,6 +134,24 @@ func NormalizeServiceName(service string) string {
 	return s
 }
 
+// TruncateUTF8 truncates the given string to make sure it uses less than limit bytes.
+// If the last character is an utf8 character that would be splitten, it removes it
+// entirely to make sure the resulting string is not broken.
+// from: https://github.com/DataDog/datadog-agent/blob/140a4ee164261ef2245340c50371ba989fbeb038/pkg/trace/traceutil/truncate.go#L34-L49
+func TruncateUTF8(s string, limit int) string {
+	if len(s) <= limit {
+		return s
+	}
+	var lastValidIndex int
+	for i := range s {
+		if i > limit {
+			return s[:lastValidIndex]
+		}
+		lastValidIndex = i
+	}
+	return s
+}
+
 // NormalizeTag applies some normalization to ensure the tags match the backend requirements.
 // Specifically used for env tag currently
 // port from: https://github.com/DataDog/datadog-agent/blob/c87e93a75b1fc97f0691faf78ae8eb2c280d6f55/pkg/trace/traceutil/normalize.go#L89

exporter/datadogexporter/utils/trace_helpers_test.go

@@ -23,6 +23,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+// ensure that truncation helper function truncates strings as expected
+// and accounts for the limit and multi byte ending characters
+// from https://github.com/DataDog/datadog-agent/blob/140a4ee164261ef2245340c50371ba989fbeb038/pkg/trace/traceutil/truncate_test.go#L15
+func TestTruncateUTF8Strings(t *testing.T) {
+	assert.Equal(t, "", TruncateUTF8("", 5))
+	assert.Equal(t, "télé", TruncateUTF8("télé", 5))
+	assert.Equal(t, "t", TruncateUTF8("télé", 2))
+	assert.Equal(t, "éé", TruncateUTF8("ééééé", 5))
+	assert.Equal(t, "ééééé", TruncateUTF8("ééééé", 18))
+	assert.Equal(t, "ééééé", TruncateUTF8("ééééé", 10))
+	assert.Equal(t, "ééé", TruncateUTF8("ééééé", 6))
+}
+
 func TestNormalizeTag(t *testing.T) {
 	for _, tt := range []struct{ in, out string }{
 		{in: "#test_starting_hash", out: "test_starting_hash"},