Skip to content

Commit

Permalink
[datadogexporter] Use resource attributes for metadata and generated …
Browse files Browse the repository at this point in the history
…metrics (#2023)

* Set hostname to running metrics from resource attributes if available

Running metrics can be duplicate in this setup but this is fine since
they are gauges.

* Use resource metadata to complet host metadata

If enabled (it is by default) the resource attributes from the first
payload from either metrics or traces will be used to populate host
metadata. If there are fields missing these will be filled in by the
exporter (this preserves behavior when not using
k8s_tagger/resourcedetection).

* Add tests for new metadata behavior

* Fix go lint

* Remove unnecessary log messages

* Improve GCP info from attributes

Most logic is taken from here: https://github.com/DataDog/datadog-agent/blob/491c309e374ed5c7f1b385b347d5f9b5ac72c2b6/pkg/util/gce/gce_tags.go#L71-L101

* Use "system" host tags for EC2 tags
This is what the Datadog Agent does https://github.com/DataDog/datadog-agent/blob/491c309e374ed5c7f1b385b347d5f9b5ac72c2b6/pkg/metadata/host/host_tags.go#L97-L139

* Add additional test for GCP hostname from attributes

* Add test for metadata.Pusher function

* Improve test coverage

* Set EC2 tags in OTel field instead of System field
This avoids clashes with the Datadog Agent
  • Loading branch information
mx-psi authored and pmatyjasek-sumo committed Apr 28, 2021
1 parent 66d59db commit 004e72c
Show file tree
Hide file tree
Showing 20 changed files with 674 additions and 136 deletions.
15 changes: 12 additions & 3 deletions exporter/datadogexporter/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

var (
errUnsetAPIKey = errors.New("api.key is not set")
errNoMetadata = errors.New("only_metadata can't be enabled when send_metadata is disabled")
errNoMetadata = errors.New("only_metadata can't be enabled when send_metadata or use_resource_metadata is disabled")
)

const (
Expand Down Expand Up @@ -166,11 +166,20 @@ type Config struct {
// OnlyMetadata defines whether to only send metadata
// This is useful for agent-collector setups, so that
// metadata about a host is sent to the backend even
// when telemetry data is reported via a different host
// when telemetry data is reported via a different host.
//
// This flag is incompatible with disabling `send_metadata`
// or `use_resource_metadata`.
OnlyMetadata bool `mapstructure:"only_metadata"`

// UseResourceMetadata defines whether to use resource attributes
// for completing host metadata (such as the hostname or host tags).
//
// By default this is true: the first resource attribute getting to
// the exporter will be used for host metadata.
// Disable this in the Collector if you are using an agent-collector setup.
UseResourceMetadata bool `mapstructure:"use_resource_metadata"`

// onceMetadata ensures only one exporter (metrics/traces) sends host metadata
onceMetadata sync.Once
}
Expand All @@ -185,7 +194,7 @@ func (c *Config) Sanitize() error {
c.TagsConfig.Env = "none"
}

if c.OnlyMetadata && !c.SendMetadata {
if c.OnlyMetadata && (!c.SendMetadata || !c.UseResourceMetadata) {
return errNoMetadata
}

Expand Down
49 changes: 26 additions & 23 deletions exporter/datadogexporter/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ func createDefaultConfig() configmodels.Exporter {
},
},

SendMetadata: true,
SendMetadata: true,
UseResourceMetadata: true,
}
}

Expand All @@ -99,23 +100,24 @@ func createMetricsExporter(
return nil, err
}

ctx, cancel := context.WithCancel(ctx)
var pushMetricsFn exporterhelper.PushMetrics

if cfg.OnlyMetadata {
pushMetricsFn = func(context.Context, pdata.Metrics) (int, error) {
// if only sending metadata ignore all metrics
pushMetricsFn = func(_ context.Context, md pdata.Metrics) (int, error) {
// only sending metadata use only metrics
once := cfg.OnceMetadata()
once.Do(func() {
attrs := pdata.NewAttributeMap()
if md.ResourceMetrics().Len() > 0 {
attrs = md.ResourceMetrics().At(0).Resource().Attributes()
}
go metadata.Pusher(ctx, params, cfg, attrs)
})
return 0, nil
}
} else {
pushMetricsFn = newMetricsExporter(params, cfg).PushMetricsData
}

ctx, cancel := context.WithCancel(ctx)
if cfg.SendMetadata {
once := cfg.OnceMetadata()
once.Do(func() {
go metadata.Pusher(ctx, params, cfg)
})
pushMetricsFn = newMetricsExporter(ctx, params, cfg).PushMetricsData
}

return exporterhelper.NewMetricsExporter(
Expand Down Expand Up @@ -148,23 +150,24 @@ func createTraceExporter(
return nil, err
}

ctx, cancel := context.WithCancel(ctx)
var pushTracesFn exporterhelper.PushTraces

if cfg.OnlyMetadata {
pushTracesFn = func(context.Context, pdata.Traces) (int, error) {
// if only sending metadata, ignore all traces
pushTracesFn = func(_ context.Context, td pdata.Traces) (int, error) {
// only sending metadata, use only attributes
once := cfg.OnceMetadata()
once.Do(func() {
attrs := pdata.NewAttributeMap()
if td.ResourceSpans().Len() > 0 {
attrs = td.ResourceSpans().At(0).Resource().Attributes()
}
go metadata.Pusher(ctx, params, cfg, attrs)
})
return 0, nil
}
} else {
pushTracesFn = newTraceExporter(params, cfg).pushTraceData
}

ctx, cancel := context.WithCancel(ctx)
if cfg.SendMetadata {
once := cfg.OnceMetadata()
once.Do(func() {
go metadata.Pusher(ctx, params, cfg)
})
pushTracesFn = newTraceExporter(ctx, params, cfg).pushTraceData
}

return exporterhelper.NewTraceExporter(
Expand Down
54 changes: 36 additions & 18 deletions exporter/datadogexporter/factory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package datadogexporter

import (
"context"
"encoding/json"
"os"
"path"
"testing"
Expand All @@ -31,6 +32,7 @@ import (
"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config"
"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata"
"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/testutils"
)

Expand Down Expand Up @@ -75,8 +77,9 @@ func TestCreateDefaultConfig(t *testing.T) {
EnvVarTags: "$DD_TAGS",
},

SendMetadata: true,
OnlyMetadata: false,
SendMetadata: true,
OnlyMetadata: false,
UseResourceMetadata: true,
}, cfg, "failed to create default config")

assert.NoError(t, configcheck.ValidateConfig(cfg))
Expand Down Expand Up @@ -132,8 +135,9 @@ func TestLoadConfig(t *testing.T) {
Endpoint: "https://trace.agent.datadoghq.eu",
},
},
SendMetadata: true,
OnlyMetadata: false,
SendMetadata: true,
OnlyMetadata: false,
UseResourceMetadata: true,
}, apiConfig)

defaultConfig := cfg.Exporters["datadog/default"].(*config.Config)
Expand Down Expand Up @@ -173,8 +177,9 @@ func TestLoadConfig(t *testing.T) {
Endpoint: "https://trace.agent.datadoghq.com",
},
},
SendMetadata: true,
OnlyMetadata: false,
SendMetadata: true,
OnlyMetadata: false,
UseResourceMetadata: true,
}, defaultConfig)

invalidConfig := cfg.Exporters["datadog/invalid"].(*config.Config)
Expand Down Expand Up @@ -256,8 +261,9 @@ func TestLoadConfigEnvVariables(t *testing.T) {
Endpoint: "https://trace.agent.datadoghq.test",
},
},
SendMetadata: true,
OnlyMetadata: false,
SendMetadata: true,
OnlyMetadata: false,
UseResourceMetadata: true,
}, apiConfig)

defaultConfig := cfg.Exporters["datadog/default2"].(*config.Config)
Expand Down Expand Up @@ -300,8 +306,9 @@ func TestLoadConfigEnvVariables(t *testing.T) {
Endpoint: "https://trace.agent.datadoghq.com",
},
},
SendMetadata: true,
OnlyMetadata: false,
SendMetadata: true,
OnlyMetadata: false,
UseResourceMetadata: true,
}, defaultConfig)
}

Expand Down Expand Up @@ -371,6 +378,8 @@ func TestCreateAPITracesExporter(t *testing.T) {
}

func TestOnlyMetadata(t *testing.T) {
server := testutils.DatadogServerMock()
defer server.Close()
logger := zap.NewNop()

factories, err := componenttest.ExampleComponents()
Expand All @@ -381,20 +390,20 @@ func TestOnlyMetadata(t *testing.T) {

ctx := context.Background()
cfg := &config.Config{
API: config.APIConfig{
Key: "notnull",
Site: "example.com",
},
SendMetadata: true,
OnlyMetadata: true,
API: config.APIConfig{Key: "notnull"},
Metrics: config.MetricsConfig{TCPAddr: confignet.TCPAddr{Endpoint: server.URL}},
Traces: config.TracesConfig{TCPAddr: confignet.TCPAddr{Endpoint: server.URL}},

SendMetadata: true,
OnlyMetadata: true,
UseResourceMetadata: true,
}

expTraces, err := factory.CreateTracesExporter(
ctx,
component.ExporterCreateParams{Logger: logger},
cfg,
)

assert.NoError(t, err)
assert.NotNil(t, expTraces)

Expand All @@ -403,7 +412,16 @@ func TestOnlyMetadata(t *testing.T) {
component.ExporterCreateParams{Logger: logger},
cfg,
)

assert.NoError(t, err)
assert.NotNil(t, expMetrics)

err = expTraces.ConsumeTraces(ctx, testutils.TestTraces.Clone())
require.NoError(t, err)

body := <-server.MetadataChan
var recvMetadata metadata.HostMetadata
err = json.Unmarshal(body, &recvMetadata)
require.NoError(t, err)
assert.Equal(t, recvMetadata.InternalHostname, "custom-hostname")

}
30 changes: 29 additions & 1 deletion exporter/datadogexporter/metadata/ec2/ec2.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package ec2

import (
"fmt"
"strings"

"github.com/aws/aws-sdk-go/aws/ec2metadata"
Expand All @@ -23,11 +24,15 @@ import (
"go.uber.org/zap"
)

var defaultPrefixes = [3]string{"ip-", "domu", "ec2amaz-"}
var (
defaultPrefixes = [3]string{"ip-", "domu", "ec2amaz-"}
ec2TagPrefix = "ec2.tag."
)

type HostInfo struct {
InstanceID string
EC2Hostname string
EC2Tags []string
}

// isDefaultHostname checks if a hostname is an EC2 default
Expand Down Expand Up @@ -95,3 +100,26 @@ func HostnameFromAttributes(attrs pdata.AttributeMap) (string, bool) {

return "", false
}

// HostInfoFromAttributes gets EC2 host info from attributes following
// OpenTelemetry semantic conventions
func HostInfoFromAttributes(attrs pdata.AttributeMap) (hostInfo *HostInfo) {
hostInfo = &HostInfo{}

if hostID, ok := attrs.Get(conventions.AttributeHostID); ok {
hostInfo.InstanceID = hostID.StringVal()
}

if hostName, ok := attrs.Get(conventions.AttributeHostName); ok {
hostInfo.EC2Hostname = hostName.StringVal()
}

attrs.ForEach(func(k string, v pdata.AttributeValue) {
if strings.HasPrefix(k, ec2TagPrefix) {
tag := fmt.Sprintf("%s:%s", strings.TrimPrefix(k, ec2TagPrefix), v.StringVal())
hostInfo.EC2Tags = append(hostInfo.EC2Tags, tag)
}
})

return
}
17 changes: 17 additions & 0 deletions exporter/datadogexporter/metadata/ec2/ec2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,20 @@ func TestHostnameFromAttributes(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, hostname, testInstanceID)
}

func TestHostInfoFromAttributes(t *testing.T) {
attrs := testutils.NewAttributeMap(map[string]string{
conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAWS,
conventions.AttributeHostID: testInstanceID,
conventions.AttributeHostName: testIP,
"ec2.tag.tag1": "val1",
"ec2.tag.tag2": "val2",
"ignored": "ignored",
})

hostInfo := HostInfoFromAttributes(attrs)

assert.Equal(t, hostInfo.InstanceID, testInstanceID)
assert.Equal(t, hostInfo.EC2Hostname, testIP)
assert.Equal(t, hostInfo.EC2Tags, []string{"tag1:val1", "tag2:val2"})
}
63 changes: 63 additions & 0 deletions exporter/datadogexporter/metadata/gcp/gcp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gcp

import (
"fmt"

"go.opentelemetry.io/collector/consumer/pdata"
"go.opentelemetry.io/collector/translator/conventions"
)

type HostInfo struct {
HostAliases []string
GCPTags []string
}

// HostnameFromAttributes gets a valid hostname from labels
// if available
func HostnameFromAttributes(attrs pdata.AttributeMap) (string, bool) {
if hostName, ok := attrs.Get(conventions.AttributeHostName); ok {
return hostName.StringVal(), true
}

return "", false
}

// HostInfoFromAttributes gets GCP host info from attributes following
// OpenTelemetry semantic conventions
func HostInfoFromAttributes(attrs pdata.AttributeMap) (hostInfo *HostInfo) {
hostInfo = &HostInfo{}

if hostID, ok := attrs.Get(conventions.AttributeHostID); ok {
// Add host id as a host alias to preserve backwards compatibility
// The Datadog Agent does not do this
hostInfo.HostAliases = append(hostInfo.HostAliases, hostID.StringVal())
hostInfo.GCPTags = append(hostInfo.GCPTags, fmt.Sprintf("instance-id:%s", hostID.StringVal()))
}

if cloudZone, ok := attrs.Get(conventions.AttributeCloudZone); ok {
hostInfo.GCPTags = append(hostInfo.GCPTags, fmt.Sprintf("zone:%s", cloudZone.StringVal()))
}

if hostType, ok := attrs.Get(conventions.AttributeHostType); ok {
hostInfo.GCPTags = append(hostInfo.GCPTags, fmt.Sprintf("instance-type:%s", hostType.StringVal()))
}

if cloudAccount, ok := attrs.Get(conventions.AttributeCloudAccount); ok {
hostInfo.GCPTags = append(hostInfo.GCPTags, fmt.Sprintf("project:%s", cloudAccount.StringVal()))
}

return
}
Loading

0 comments on commit 004e72c

Please sign in to comment.