Bump to opentelemetry-mapping-go v0.11.0

.copyright-overrides.yml

@@ -160,6 +160,8 @@ github.com/dgraph-io/badger:
  - "Copyright 2020 Dgraph Labs, Inc. and Contributors"
  - "Copyright 2021 Dgraph Labs, Inc. and Contributors"
 github.com/mostynb/go-grpc-compression: "Copyright 2023 Mostyn Bramley-Moore."
+github.com/pjbgf/sha1cd: "Copyright 2009 The Go Authors"
+github.com/stormcat24/protodep: "Copyright Akinori Yamada"
 
 # These have large AUTHORS/CONTRIBUTORS files but attribute copyright as a group
 github.com/aptly-dev/aptly: Copyright 2013-2015 aptly authors. All rights reserved.

LICENSE-3rdparty.csv

@@ -1445,6 +1445,8 @@ core,github.com/spf13/cobra,Apache-2.0,Copyright © 2013 Steve Francia <spf@spf1
 core,github.com/spf13/jwalterweatherman,MIT,Copyright (c) 2014 Steve Francia
 core,github.com/spf13/pflag,BSD-3-Clause,Copyright (c) 2012 Alex Ogier. All rights reserved | Copyright (c) 2012 The Go Authors. All rights reserved
 core,github.com/stoewer/go-strcase,MIT,"Copyright (c) 2017, Adrian Stoewer <adrian.stoewer@rz.ifi.lmu.de>"
+core,github.com/stormcat24/protodep,Apache-2.0,Copyright Akinori Yamada
+core,github.com/stormcat24/protodep/pkg/logger,Apache-2.0,Copyright Akinori Yamada
 core,github.com/streadway/amqp,BSD-2-Clause,"Copyright (c) 2012-2019, Sean Treadway, SoundCloud Ltd"
 core,github.com/stretchr/objx,MIT,"Copyright (c) 2014 Stretchr, Inc | Copyright (c) 2017-2018 objx contributors"
 core,github.com/stretchr/testify/assert,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"

comp/core/log/go.mod

@@ -56,19 +56,28 @@ require (
 	github.com/DataDog/datadog-go/v5 v5.1.1 // indirect
 	github.com/DataDog/go-sqllexer v0.0.9 // indirect
 	github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.11.0 // indirect
 	github.com/DataDog/viper v1.12.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/outcaste-io/ristretto v0.2.1 // indirect
 	github.com/pelletier/go-toml v1.2.0 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
@@ -84,15 +93,27 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/collector/component v0.91.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.91.0 // indirect
+	go.opentelemetry.io/collector/confmap v0.91.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.0.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.0.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.91.0 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.23.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect
 	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

comp/otelcol/otlp/internal/logsagentexporter/factory.go

@@ -12,6 +12,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/logs/agent/config"
 	"github.com/DataDog/datadog-agent/pkg/logs/message"
 	"github.com/DataDog/datadog-agent/pkg/logs/sources"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"go.opentelemetry.io/collector/component"
 	exp "go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
@@ -46,15 +47,26 @@ func (f *factory) createLogsExporter(
 ) (exp.Logs, error) {
 	logSource := sources.NewLogSource(logSourceName, &config.LogsConfig{})
 
+	// TODO: Ideally the attributes translator would be created once and reused
+	// across all signals. This would need unifying the logsagent and serializer
+	// exporters into a single exporter.
+	attributesTranslator, err := attributes.NewTranslator(set.TelemetrySettings)
+	if err != nil {
+		return nil, err
+	}
+
+	exporter, err := newExporter(set.TelemetrySettings, logSource, f.logsAgentChannel, attributesTranslator)
+	if err != nil {
+		return nil, err
+	}
+
 	ctx, cancel := context.WithCancel(ctx)
 	// cancel() runs on shutdown
-	pusher := createConsumeLogsFunc(set.TelemetrySettings.Logger, logSource, f.logsAgentChannel)
-
 	return exporterhelper.NewLogsExporter(
 		ctx,
 		set,
 		c,
-		pusher,
+		exporter.ConsumeLogs,
 		exporterhelper.WithShutdown(func(context.Context) error {
 			cancel()
 			return nil

comp/otelcol/otlp/internal/logsagentexporter/logs_exporter.go

@@ -14,79 +14,85 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/logs/message"
 	"github.com/DataDog/datadog-agent/pkg/logs/sources"
 	"github.com/DataDog/datadog-agent/pkg/util/scrubber"
-	"go.uber.org/zap"
+	"github.com/stormcat24/protodep/pkg/logger"
 
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	logsmapping "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs"
+	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/pdata/plog"
 )
 
-// otelTag specifies a tag to be added to all logs sent from the Datadog Agent
-const otelTag = "otel_source:datadog_agent"
+// otelSource specifies a source to be added to all logs sent from the Datadog Agent.
+// The tag has key `otel_source` and the value specified on this constant.
+const otelSource = "datadog_agent"
 
-// createConsumeLogsFunc returns an implementation of consumer.ConsumeLogsFunc
-func createConsumeLogsFunc(logger *zap.Logger, logSource *sources.LogSource, logsAgentChannel chan *message.Message) func(context.Context, plog.Logs) error {
-
-	return func(_ context.Context, ld plog.Logs) (err error) {
-		defer func() {
-			if err != nil {
-				newErr, scrubbingErr := scrubber.ScrubString(err.Error())
-				if scrubbingErr != nil {
-					err = scrubbingErr
-				} else {
-					err = errors.New(newErr)
-				}
-			}
-		}()
+type exporter struct {
+	set              component.TelemetrySettings
+	logsAgentChannel chan *message.Message
+	logSource        *sources.LogSource
+	translator       *logsmapping.Translator
+}
 
-		rsl := ld.ResourceLogs()
-		// Iterate over resource logs
-		for i := 0; i < rsl.Len(); i++ {
-			rl := rsl.At(i)
-			sls := rl.ScopeLogs()
-			res := rl.Resource()
-			for j := 0; j < sls.Len(); j++ {
-				sl := sls.At(j)
-				lsl := sl.LogRecords()
-				// iterate over Logs
-				for k := 0; k < lsl.Len(); k++ {
-					log := lsl.At(k)
-					ddLog := logsmapping.Transform(log, res, logger)
+func newExporter(
+	set component.TelemetrySettings,
+	logSource *sources.LogSource,
+	logsAgentChannel chan *message.Message,
+	attributesTranslator *attributes.Translator,
+) (*exporter, error) {
+	translator, err := logsmapping.NewTranslator(set, attributesTranslator, otelSource)
+	if err != nil {
+		return nil, err
+	}
 
-					var tags []string
-					if ddTags := ddLog.GetDdtags(); ddTags == "" {
-						tags = []string{otelTag}
-					} else {
-						tags = append(strings.Split(ddTags, ","), otelTag)
-					}
-					// Tags are set in the message origin instead
-					ddLog.Ddtags = nil
-					service := ""
-					if ddLog.Service != nil {
-						service = *ddLog.Service
-					}
-					status := ddLog.AdditionalProperties["status"]
-					if status == "" {
-						status = message.StatusInfo
-					}
-					origin := message.NewOrigin(logSource)
-					origin.SetTags(tags)
-					origin.SetService(service)
-					origin.SetSource(logSourceName)
+	return &exporter{
+		set:              set,
+		logsAgentChannel: logsAgentChannel,
+		logSource:        logSource,
+		translator:       translator,
+	}, nil
+}
 
-					content, err := ddLog.MarshalJSON()
-					if err != nil {
-						logger.Error("Error parsing log: " + err.Error())
-					}
+func (e *exporter) ConsumeLogs(ctx context.Context, ld plog.Logs) (err error) {
+	defer func() {
+		if err != nil {
+			newErr, scrubbingErr := scrubber.ScrubString(err.Error())
+			if scrubbingErr != nil {
+				err = scrubbingErr
+			} else {
+				err = errors.New(newErr)
+			}
+		}
+	}()
 
-					// ingestionTs is an internal field used for latency tracking on the status page, not the actual log timestamp.
-					ingestionTs := time.Now().UnixNano()
-					message := message.NewMessage(content, origin, status, ingestionTs)
+	payloads := e.translator.MapLogs(ctx, ld)
+	for _, ddLog := range payloads {
+		tags := strings.Split(ddLog.GetDdtags(), ",")
+		// Tags are set in the message origin instead
+		ddLog.Ddtags = nil
+		service := ""
+		if ddLog.Service != nil {
+			service = *ddLog.Service
+		}
+		status := ddLog.AdditionalProperties["status"]
+		if status == "" {
+			status = message.StatusInfo
+		}
+		origin := message.NewOrigin(e.logSource)
+		origin.SetTags(tags)
+		origin.SetService(service)
+		origin.SetSource(logSourceName)
 
-					logsAgentChannel <- message
-				}
-			}
+		content, err := ddLog.MarshalJSON()
+		if err != nil {
+			logger.Error("Error parsing log: " + err.Error())
 		}
 
-		return nil
+		// ingestionTs is an internal field used for latency tracking on the status page, not the actual log timestamp.
+		ingestionTs := time.Now().UnixNano()
+		message := message.NewMessage(content, origin, status, ingestionTs)
+
+		e.logsAgentChannel <- message
 	}
+
+	return nil
 }

comp/otelcol/otlp/internal/serializerexporter/exporter.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics"
 	"go.opentelemetry.io/collector/component"
@@ -80,7 +81,7 @@ type exporter struct {
 	cardinality collectors.TagCardinality
 }
 
-func translatorFromConfig(set component.TelemetrySettings, cfg *exporterConfig) (*metrics.Translator, error) {
+func translatorFromConfig(set component.TelemetrySettings, attributesTranslator *attributes.Translator, cfg *exporterConfig) (*metrics.Translator, error) {
 	histogramMode := metrics.HistogramMode(cfg.Metrics.HistConfig.Mode)
 	switch histogramMode {
 	case metrics.HistogramModeCounters, metrics.HistogramModeNoBuckets, metrics.HistogramModeDistributions:
@@ -127,16 +128,16 @@ func translatorFromConfig(set component.TelemetrySettings, cfg *exporterConfig)
 	options = append(options, metrics.WithInitialCumulMonoValueMode(
 		metrics.InitialCumulMonoValueMode(cfg.Metrics.SumConfig.InitialCumulativeMonotonicMode)))
 
-	return metrics.NewTranslator(set, options...)
+	return metrics.NewTranslator(set, attributesTranslator, options...)
 }
 
-func newExporter(set component.TelemetrySettings, s serializer.MetricSerializer, cfg *exporterConfig) (*exporter, error) {
+func newExporter(set component.TelemetrySettings, attributesTranslator *attributes.Translator, s serializer.MetricSerializer, cfg *exporterConfig) (*exporter, error) {
 	// Log any warnings from unmarshaling.
 	for _, warning := range cfg.warnings {
 		set.Logger.Warn(warning)
 	}
 
-	tr, err := translatorFromConfig(set, cfg)
+	tr, err := translatorFromConfig(set, attributesTranslator, cfg)
 	if err != nil {
 		return nil, fmt.Errorf("incorrect OTLP metrics configuration: %w", err)
 	}

comp/otelcol/otlp/internal/serializerexporter/exporter_test.go

@@ -10,7 +10,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/exporter/exportertest"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 
 	"github.com/DataDog/datadog-agent/pkg/config"
@@ -190,19 +192,17 @@ func Test_ConsumeMetrics_Tags(t *testing.T) {
 				tt.setConfig(t)
 			}
 			rec := &metricRecorder{}
-			exp, err := newExporter(
-				componenttest.NewNopTelemetrySettings(),
-				rec,
-				NewFactory(rec).CreateDefaultConfig().(*exporterConfig),
+			ctx := context.Background()
+			f := NewFactory(rec)
+			exp, err := f.CreateMetricsExporter(
+				ctx,
+				exportertest.NewNopCreateSettings(),
+				f.CreateDefaultConfig(),
 			)
-			if err != nil {
-				t.Errorf("newExporter() returns unexpected error: %v", err)
-				return
-			}
-			if err := exp.ConsumeMetrics(context.Background(), tt.genMetrics(t)); err != nil {
-				t.Errorf("ConsumeMetrics() returns unexpected error: %v", err)
-				return
-			}
+			require.NoError(t, err)
+			require.NoError(t, exp.Start(ctx, componenttest.NewNopHost()))
+			require.NoError(t, exp.ConsumeMetrics(ctx, tt.genMetrics(t)))
+			require.NoError(t, exp.Shutdown(ctx))
 
 			if tt.wantSketchTags.Len() > 0 {
 				assert.Equal(t, tt.wantSketchTags, rec.sketchSeriesList[0].Tags)

comp/otelcol/otlp/internal/serializerexporter/factory.go

@@ -8,6 +8,7 @@ package serializerexporter
 import (
 	"context"
 
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/consumer"
@@ -41,7 +42,15 @@ func NewFactory(s serializer.MetricSerializer) exp.Factory {
 func (f *factory) createMetricExporter(ctx context.Context, params exp.CreateSettings, c component.Config) (exp.Metrics, error) {
 	cfg := c.(*exporterConfig)
 
-	newExp, err := newExporter(params.TelemetrySettings, f.s, cfg)
+	// TODO: Ideally the attributes translator would be created once and reused
+	// across all signals. This would need unifying the logsagent and serializer
+	// exporters into a single exporter.
+	attributesTranslator, err := attributes.NewTranslator(params.TelemetrySettings)
+	if err != nil {
+		return nil, err
+	}
+
+	newExp, err := newExporter(params.TelemetrySettings, attributesTranslator, f.s, cfg)
 	if err != nil {
 		return nil, err
 	}

comp/trace/config/setup.go

@@ -28,6 +28,8 @@ import (
 	rc "github.com/DataDog/datadog-agent/pkg/config/remote/client"
 	"github.com/DataDog/datadog-agent/pkg/config/remote/data"
 	"github.com/DataDog/datadog-agent/pkg/config/utils"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
+	"go.opentelemetry.io/collector/component/componenttest"
 
 	//nolint:revive // TODO(APM) Fix revive linter
 	configUtils "github.com/DataDog/datadog-agent/pkg/config/utils"
@@ -345,13 +347,22 @@ func applyDatadogConfig(c *config.AgentConfig, core corecompcfg.Component) error
 	if otlp.IsEnabled(coreconfig.Datadog) {
 		grpcPort = core.GetInt(coreconfig.OTLPTracePort)
 	}
+
+	// We use a noop set of telemetry settings. This silences all warnings and metrics from the attributes translator.
+	// The Datadog exporter overrides this with its own attributes translator using its own telemetry settings.
+	attributesTranslator, err := attributes.NewTranslator(componenttest.NewNopTelemetrySettings())
+	if err != nil {
+		return err
+	}
+
 	c.OTLPReceiver = &config.OTLP{
 		BindHost:               c.ReceiverHost,
 		GRPCPort:               grpcPort,
 		MaxRequestBytes:        c.MaxRequestBytes,
 		SpanNameRemappings:     coreconfig.Datadog.GetStringMapString("otlp_config.traces.span_name_remappings"),
 		SpanNameAsResourceName: core.GetBool("otlp_config.traces.span_name_as_resource_name"),
 		ProbabilisticSampling:  core.GetFloat64("otlp_config.traces.probabilistic_sampler.sampling_percentage"),
+		AttributesTranslator:   attributesTranslator,
 	}
 
 	if core.IsSet("apm_config.install_id") {