[datadogexporter] Send host metadata (#1351)

* Add EC2 metadata

* Add option to disable sending metadata
Set it to false in tests to avoid creating goroutines

* Launch host metadata goroutine only once

* Add tests for EC2 hostname resolution

* Address linter issues

* [empty] Retrigger CI

* Do not send tags with metrics or traces
The backend will add these tags since they are sent with metadata

* Add env to host tags
Reusing the `GetTags` function for this since this PR leaves it without use

* Fix indentation on go.mod

* Apply suggestions from code review

Fix documentation comments

Co-authored-by: Kylian Serrania <kylian.serrania@datadoghq.com>

* Use params.ApplicationStartInfo for getting Flavor and Version

* [empty] Retrigger CI

* [empty] Retrigger CI again

* [empty] Retrigger CI (third attempt)

* [empty] Retrigger CI (4)

* Fix after merge

* Improve test coverage

Co-authored-by: Kylian Serrania <kylian.serrania@datadoghq.com>

exporter/datadogexporter/config/config.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 
 	"go.opentelemetry.io/collector/config/configmodels"
 	"go.opentelemetry.io/collector/config/confignet"
@@ -99,28 +100,12 @@ type TagsConfig struct {
 	Tags []string `mapstructure:"tags"`
 }
 
-// GetTags gets the default tags extracted from the configuration
-func (t *TagsConfig) GetTags(addHost bool) []string {
-	tags := make([]string, 0, 4)
-
-	vars := map[string]string{
-		"env":     t.Env,
-		"service": t.Service,
-		"version": t.Version,
-	}
-
-	if addHost {
-		vars["host"] = t.Hostname
+// GetHostTags gets the host tags extracted from the configuration
+func (t *TagsConfig) GetHostTags() []string {
+	tags := t.Tags
+	if t.Env != "none" {
+		tags = append(tags, fmt.Sprintf("env:%s", t.Env))
 	}
-
-	for name, val := range vars {
-		if val != "" {
-			tags = append(tags, fmt.Sprintf("%s:%s", name, val))
-		}
-	}
-
-	tags = append(tags, t.Tags...)
-
 	return tags
 }
 
@@ -138,6 +123,16 @@ type Config struct {
 
 	// Traces defines the Traces exporter specific configuration
 	Traces TracesConfig `mapstructure:"traces"`
+
+	// SendMetadata defines whether to send host metadata
+	SendMetadata bool `mapstructure:"send_metadata"`
+
+	// onceMetadata ensures only one exporter (metrics/traces) sends host metadata
+	onceMetadata sync.Once
+}
+
+func (c *Config) OnceMetadata() *sync.Once {
+	return &c.onceMetadata
 }
 
 // Sanitize tries to sanitize a given configuration

exporter/datadogexporter/config/config_test.go

@@ -22,25 +22,23 @@ import (
 	"go.opentelemetry.io/collector/config/confignet"
 )
 
-func TestTags(t *testing.T) {
+func TestHostTags(t *testing.T) {
 	tc := TagsConfig{
 		Hostname: "customhost",
 		Env:      "customenv",
-		Service:  "customservice",
-		Version:  "customversion",
-		Tags:     []string{"key1:val1", "key2:val2"},
+		// Service and version should be only used for traces
+		Service: "customservice",
+		Version: "customversion",
+		Tags:    []string{"key1:val1", "key2:val2"},
 	}
 
 	assert.ElementsMatch(t,
 		[]string{
-			"host:customhost",
 			"env:customenv",
-			"service:customservice",
-			"version:customversion",
 			"key1:val1",
 			"key2:val2",
 		},
-		tc.GetTags(true), // get host
+		tc.GetHostTags(),
 	)
 }
 

exporter/datadogexporter/factory.go

@@ -23,6 +23,7 @@ import (
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata"
 )
 
 const (
@@ -65,12 +66,14 @@ func createDefaultConfig() configmodels.Exporter {
 				Endpoint: "", // set during config sanitization
 			},
 		},
+
+		SendMetadata: true,
 	}
 }
 
 // createMetricsExporter creates a metrics exporter based on this config.
 func createMetricsExporter(
-	_ context.Context,
+	ctx context.Context,
 	params component.ExporterCreateParams,
 	c configmodels.Exporter,
 ) (component.MetricsExporter, error) {
@@ -82,22 +85,34 @@ func createMetricsExporter(
 		return nil, err
 	}
 
-	exp, err := newMetricsExporter(params.Logger, cfg)
+	exp, err := newMetricsExporter(params, cfg)
 	if err != nil {
 		return nil, err
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	if cfg.SendMetadata {
+		once := cfg.OnceMetadata()
+		once.Do(func() {
+			go metadata.Pusher(ctx, params, cfg)
+		})
+	}
+
 	return exporterhelper.NewMetricsExporter(
 		cfg,
 		exp.PushMetricsData,
 		exporterhelper.WithQueue(exporterhelper.CreateDefaultQueueSettings()),
 		exporterhelper.WithRetry(exporterhelper.CreateDefaultRetrySettings()),
+		exporterhelper.WithShutdown(func(context.Context) error {
+			cancel()
+			return nil
+		}),
 	)
 }
 
 // createTraceExporter creates a trace exporter based on this config.
 func createTraceExporter(
-	_ context.Context,
+	ctx context.Context,
 	params component.ExporterCreateParams,
 	c configmodels.Exporter,
 ) (component.TraceExporter, error) {
@@ -109,13 +124,25 @@ func createTraceExporter(
 		return nil, err
 	}
 
-	exp, err := newTraceExporter(params.Logger, cfg)
+	exp, err := newTraceExporter(params, cfg)
 	if err != nil {
 		return nil, err
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	if cfg.SendMetadata {
+		once := cfg.OnceMetadata()
+		once.Do(func() {
+			go metadata.Pusher(ctx, params, cfg)
+		})
+	}
+
 	return exporterhelper.NewTraceExporter(
 		cfg,
 		exp.pushTraceData,
+		exporterhelper.WithShutdown(func(context.Context) error {
+			cancel()
+			return nil
+		}),
 	)
 }

exporter/datadogexporter/factory_test.go

@@ -48,6 +48,7 @@ func TestCreateDefaultConfig(t *testing.T) {
 		Traces: config.TracesConfig{
 			SampleRate: 1,
 		},
+		SendMetadata: true,
 	}, cfg, "failed to create default config")
 
 	assert.NoError(t, configcheck.ValidateConfig(cfg))
@@ -101,6 +102,7 @@ func TestLoadConfig(t *testing.T) {
 				Endpoint: "https://trace.agent.datadoghq.eu",
 			},
 		},
+		SendMetadata: true,
 	}, apiConfig)
 
 	invalidConfig2 := cfg.Exporters["datadog/invalid"].(*config.Config)
@@ -128,6 +130,7 @@ func TestCreateAPIMetricsExporter(t *testing.T) {
 	// Use the mock server for API key validation
 	c := (cfg.Exporters["datadog/api"]).(*config.Config)
 	c.Metrics.TCPAddr.Endpoint = server.URL
+	c.SendMetadata = false
 	cfg.Exporters["datadog/api"] = c
 
 	ctx := context.Background()
@@ -160,7 +163,7 @@ func TestCreateAPITracesExporter(t *testing.T) {
 	// Use the mock server for API key validation
 	c := (cfg.Exporters["datadog/api"]).(*config.Config)
 	c.Metrics.TCPAddr.Endpoint = server.URL
-	cfg.Exporters["datadog/api"] = c
+	c.SendMetadata = false
 
 	ctx := context.Background()
 	exp, err := factory.CreateTraceExporter(

exporter/datadogexporter/go.mod

@@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datado
 go 1.14
 
 require (
+	github.com/aws/aws-sdk-go v1.34.9
 	github.com/DataDog/datadog-agent/pkg/trace/exportable v0.0.0-20201016145401-4646cf596b02
 	github.com/gogo/protobuf v1.3.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible

exporter/datadogexporter/metadata/ec2/ec2.go

@@ -0,0 +1,80 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package ec2
+
+import (
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"go.uber.org/zap"
+)
+
+var defaultPrefixes = [3]string{"ip-", "domu", "ec2amaz-"}
+
+type HostInfo struct {
+	InstanceID  string
+	EC2Hostname string
+}
+
+// isDefaultHostname checks if a hostname is an EC2 default
+func isDefaultHostname(hostname string) bool {
+	for _, val := range defaultPrefixes {
+		if strings.HasPrefix(hostname, val) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// GetHostInfo gets the hostname info from EC2 metadata
+func GetHostInfo(logger *zap.Logger) (hostInfo *HostInfo) {
+	sess, err := session.NewSession()
+	hostInfo = &HostInfo{}
+
+	if err != nil {
+		logger.Warn("Failed to build AWS session", zap.Error(err))
+		return
+	}
+
+	meta := ec2metadata.New(sess)
+
+	if !meta.Available() {
+		logger.Info("EC2 Metadata not available")
+		return
+	}
+
+	if idDoc, err := meta.GetInstanceIdentityDocument(); err == nil {
+		hostInfo.InstanceID = idDoc.InstanceID
+	} else {
+		logger.Warn("Failed to get EC2 instance id document", zap.Error(err))
+	}
+
+	if ec2Hostname, err := meta.GetMetadata("hostname"); err == nil {
+		hostInfo.EC2Hostname = ec2Hostname
+	} else {
+		logger.Warn("Failed to get EC2 hostname", zap.Error(err))
+	}
+
+	return
+}
+
+func (hi *HostInfo) GetHostname(logger *zap.Logger) string {
+	if isDefaultHostname(hi.EC2Hostname) {
+		return hi.InstanceID
+	}
+
+	return hi.EC2Hostname
+}

exporter/datadogexporter/metadata/ec2/ec2_test.go

@@ -0,0 +1,52 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package ec2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+)
+
+const (
+	testIP         = "ip-12-34-56-78.us-west-2.compute.internal"
+	testDomu       = "domu-12-34-56-78.us-west-2.compute.internal"
+	testEC2        = "ec2amaz-12-34-56-78.us-west-2.compute.internal"
+	customHost     = "custom-hostname"
+	testInstanceID = "i-0123456789"
+)
+
+func TestDefaultHostname(t *testing.T) {
+	assert.True(t, isDefaultHostname(testIP))
+	assert.True(t, isDefaultHostname(testDomu))
+	assert.True(t, isDefaultHostname(testEC2))
+	assert.False(t, isDefaultHostname(customHost))
+}
+
+func TestGetHostname(t *testing.T) {
+	logger := zap.NewNop()
+
+	hostInfo := &HostInfo{
+		InstanceID:  testInstanceID,
+		EC2Hostname: testIP,
+	}
+	assert.Equal(t, testInstanceID, hostInfo.GetHostname(logger))
+
+	hostInfo = &HostInfo{
+		InstanceID:  testInstanceID,
+		EC2Hostname: customHost,
+	}
+	assert.Equal(t, customHost, hostInfo.GetHostname(logger))
+}

exporter/datadogexporter/metadata/host.go

@@ -18,14 +18,18 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/ec2"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/system"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/valid"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/utils/cache"
 )
 
 // GetHost gets the hostname according to configuration.
-// It gets the configuration hostname and if
-// not available it relies on the OS hostname
+// It checks in the following order
+// 1. Cache
+// 2. Configuration
+// 3. EC2 instance metadata
+// 4. System
 func GetHost(logger *zap.Logger, cfg *config.Config) *string {
 	if cacheVal, ok := cache.Cache.Get(cache.CanonicalHostnameKey); ok {
 		return cacheVal.(*string)
@@ -38,9 +42,14 @@ func GetHost(logger *zap.Logger, cfg *config.Config) *string {
 		logger.Error("Hostname set in configuration is invalid", zap.Error(err))
 	}
 
-	// Get system hostname
-	hostInfo := system.GetHostInfo(logger)
-	hostname := hostInfo.GetHostname(logger)
+	ec2Info := ec2.GetHostInfo(logger)
+	hostname := ec2Info.GetHostname(logger)
+
+	if hostname == "" {
+		// Get system hostname
+		systemInfo := system.GetHostInfo(logger)
+		hostname = systemInfo.GetHostname(logger)
+	}
 
 	if err := valid.Hostname(hostname); err != nil {
 		// If invalid log but continue